diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 23d236d6..57640182 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,35 +1,29 @@ --- name: Bug report -about: Create a report to help us improve +about: Create a report to report a problem that needs to be fixed +labels: bug +title: "BUG: " --- -**Describe the bug** -A clear and concise description of what the bug is. +# Description +A clear and concise description of what the bug is, including a description +of what you expected the outcome to be. -**To Reproduce** +# To Reproduce this bug: Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -or +Consider including images or test files to help others reproduce the bug and +solve the problem. -``` -# test code here -``` - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Version [e.g. Python 3.7] +## Test configuration + - OS: [e.g. Hal] + - Version [e.g. Python 3.47] - Other details about your setup that could be relevant -**Additional context** -Add any other context about the problem here. +# Additional context +Add any other context about the problem here, including expected behaviour. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 066b2d92..d02da2ef 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,17 +1,27 @@ --- name: Feature request about: Suggest an idea for this project +title: "ENH: " +labels: enhancement --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +# Description +A clear and concise description of the new feature or behaviour you would like. -**Describe the solution you'd like** +## Potential impact + +- Is the feature related to an existing problem? +- How critical is this feature to your workflow? +- How wide of an impact to you anticipate this enhancement having? +- Would this break any existing functionality? + +## Potential solution(s) A clear and concise description of what you want to happen. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +# Alternatives +A clear description of any alternative solutions or features you've considered. -**Additional context** -Add any other context or screenshots about the feature request here. +# Additional context +Add any other context or screenshots about the feature request here, potentially +including your operational configuration. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 00000000..da43edc7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,19 @@ +--- +name: Question +about: A question about this project +title: "QUEST: " +labels: question + +--- + +# Description +A clear and concise summary of your query + +## Example code (optional) +If relevant, include sample code, images, or files so that others can understand +the full context of your question. + +## Configuration + - OS: [e.g. Hal] + - Version: [e.g. Python 3.47] + - Other details about your setup that could be relevant diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5331eb30..e1d2dbe5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,12 @@ # Description -Addresses # (issue) +Addresses #(issue) Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. Please see ``CONTRIBUTING.md`` for more guidelines. -## Type of change +# Type of change Please delete options that are not relevant. @@ -25,9 +25,9 @@ your test configuration - Test A - Test B -## Test Configuration -* Operating system: [Os Type] -* Version number: [Python 2.9] +**Test Configuration**: +* Operating system: Hal +* Version number: Python 3.X * Any details about your local setup that are relevant # Checklist: @@ -43,3 +43,6 @@ your test configuration - [ ] Any dependent changes have been merged and published in downstream modules - [ ] Add a note to ``CHANGELOG.md``, summarizing the changes - [ ] Update zenodo.json file for new code contributors + +If this is a release PR, replace the first item of the above checklist with the release +checklist on the wiki: https://github.com/pysat/pysat/wiki/Checklist-for-Release diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dffe178e..2b6d7e88 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and check the sphinx build, links in the docs, and the readability of the zenodo file # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Documentation Check @@ -8,11 +8,11 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ["ubuntu-latest"] strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: ["3.11"] # Keep this version at the highest supported Python version name: Documentation tests steps: @@ -25,8 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r test_requirements.txt - pip install -r requirements.txt + pip install .[doc] - name: Set up pysat run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be920387..61125636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,13 +10,19 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.9", "3.10"] - numpy_ver: [latest] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.10", "3.11"] + numpy_ver: ["latest"] + test_config: ["latest"] include: - - python-version: "3.8" - numpy_ver: "1.20" + - python-version: "3.9" + numpy_ver: "1.21" os: ubuntu-latest + test_config: "NEP29" + - python-version: "3.6.8" + numpy_ver: "1.19.5" + os: "ubuntu-20.04" + test_config: "Ops" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -31,18 +37,25 @@ jobs: if: ${{ matrix.os == 'macos-latest' }} run: brew reinstall gcc + - name: Install Operational dependencies + if: ${{ matrix.test_config == 'Ops'}} + run: | + pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + pip install "cdflib<1.0" + pip install -r requirements.txt + pip install -r test_requirements.txt + pip install . + - name: Install NEP29 dependencies - if: ${{ matrix.numpy_ver != 'latest'}} + if: ${{ matrix.test_config == 'NEP29'}} run: | - pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - # Need to force a version of pandas compliant with NEP29 - pip install "pandas<1.5" + pip install numpy==${{ matrix.numpy_ver }} + pip install --upgrade-strategy only-if-needed .[test] - name: Install standard dependencies + if: ${{ matrix.test_config == 'latest'}} run: | - pip install -r requirements.txt - pip install pysatCDF --no-binary=pysatCDF - pip install -r test_requirements.txt + pip install .[test] - name: Set up pysat run: | @@ -56,10 +69,9 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: | - pytest -vs --cov=pysatNASA/ + run: pytest - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --rcfile=setup.cfg --service=github + run: coveralls --rcfile=pyproject.toml --service=github diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml new file mode 100644 index 00000000..cc60a7dc --- /dev/null +++ b/.github/workflows/pip_rc_install.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies and the latest RC of pysatNASA from test pypi. +# This test should be manually run before a pysatNASA RC is officially approved and versioned. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test install of latest RC from pip + +on: [workflow_dispatch] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.11"] # Keep this version at the highest supported Python version + + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install standard dependencies + run: pip install -r requirements.txt + + - name: Install pysatNASA RC + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysatNASA + + - name: Set up pysat + run: | + mkdir pysatData + python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" + + - name: Check that install imports correctly + run: | + cd .. + python -c "import pysatNASA; print(pysatNASA.__version__)" diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index ec9e4d20..e0ab0742 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -1,4 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and the latest RC of pysat from test pypi. +# All unit tests for pysatNASA will be run using the pysat RC. +# This test should be manually run before a pysat RC is officially approved and versioned. # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test with latest pysat RC @@ -10,8 +12,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.11"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -27,14 +29,12 @@ jobs: run: brew reinstall gcc - name: Install pysat RC - run: | - pip install -r test_requirements.txt - pip install --no-deps -i https://test.pypi.org/simple/ pysat + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - name: Install standard dependencies run: | pip install -r requirements.txt - pip install pysatCDF --no-binary=pysatCDF + pip install -r test_requirements.txt - name: Set up pysat run: | diff --git a/.gitignore b/.gitignore index 6db464db..c8350919 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ custom_lint.sh # IDEs .idea/ + +# vscode +.vscode diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..23fa5ade --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,22 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/.zenodo.json b/.zenodo.json index a5bcff9a..1b5b3db1 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -20,6 +20,11 @@ "name": "Smith, Jonathon", "orcid": "0000-0002-8191-4765" }, + { + "affilitation":"University of Colorado at Boulder", + "name": "Navarro, Luis", + "orcid": "0000-0002-6362-6575" + }, { "affiliation": "Predictive Science", "name": "Pembroke, Asher" diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a1d903..0bdc06a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,64 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## [0.0.5] - 2023-06-27 +* New Instruments + * ACE EPAM + * ACE MAG + * ACE SIS + * ACE SWEPAM + * DE2 Fabry-Perot Interferometer (FPI) + * DE2 Vector Electric Field Instrument (VEFI) and magnetometer + * DMSP SSUSI EDR-Aurora data + * IGS GPS (TEC and ROTI) + * SES-14 GOLD -- tdisk, tlimb and o2den data products added + * TIMED GUVI +* Bug Fixes + * Pandas datasets made with cdflib now have header level meta + * Updated CDAWeb routines to allow for data stored by year/day-of-year + * Updated GOLD nmax to sort scans by time. + * Added 1 usec to GOLD nmax channel B times to ensure timestamp uniqueness + * Fixed multi-file loads for cdf xarray datasets. + * Adds a 0.1 sec delay between file downloads to avoid excessive calls + to servers. +* Documentation + * Added missing sub-module imports + * Added discussion of ICON constellation to docstrings, including caveats +* Enhancements + * Added CDAWeb methods that can use `cdasws` to get the remote file list + * Updated platform methods to follow a consistent style and work with the + general `init` function + * Added unit tests for the different platform method attributes + * xarray support for TIMED SABER and SEE + * Added `drop_dims` kwarg to `load_xarray` interface so that orphan dims can + be removed before attempting to merge. + * Added `var_translation` kwarg to `load_xarray` interface so that variables can + be renamed before attempting to merge. + * Improved usage of cdflib for users in xarray instruments + * Added a generalized `clean` routine to replace fill vals with NaNs +* Deprecations + * Deprecated jpl_gps instrument module, moved roti instrument to igs_gps +* Maintenance + * Updated download functions to take data_path as an arg, not a kwarg + * Removed duplicate tests if pysatCDF not installed + * Removed pysatCDF tests on GitHub Actions workflows (see #167) + * Updated actions and templates based on pysatEcosystem docs + * Remove pandas cap on NEP29 tests + * Updated docstring style for consistency + * Removed version cap for xarray + * Added manual workflow to check that latest RC is installable through test pip + * Update meta label type for instruments + * Updated GitHub Actions workflows for improved compliance with pip>=23.0 + * Added .readthedocs.yml to configure settings there. + * Use pyproject.toml to manage installation and metadata + * Set use_cdflib=True for supported xarray instruments + * Set pysat 3.1.0 minimum + * Use pysat logger to raise non-deprecation warnings + * Update syntax based on latest pysat deprecations to make the code compatible with pysat 3.2.0. + * Updated syntax compliance with cdflib 1.0+ + * Updated use of `decode_times` kwarg when loading xarray data to maintain current behaviour + + ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e9b2a04..45943ed4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,33 +78,44 @@ To set up `pysatNASA` for local development: 4. When you're done making changes, run all the checks to ensure that nothing - is broken on your local system, as well as check for flake8 compliance: + is broken on your local system: - ``` - pytest -vs --flake8 pysatNASA - ``` + ``` + pytest pysatNASA + ``` + +5. You should also check for flake8 style compliance: + + ``` + flake8 . --count --select=D,E,F,H,W --show-source --statistics + ``` -5. Update/add documentation (in ``docs``), if relevant + Note that pysat uses the `flake-docstrings` and `hacking` packages to ensure + standards in docstring formatting. -6. Add your name to the .zenodo.json file as an author -7. Commit your changes: - ``` - git add . - git commit -m "AAA: Brief description of your changes" - ``` - Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). - `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), - see the discussion there for a full list of this shorthand notation. +6. Update/add documentation (in ``docs``), if relevant -8. Once you are happy with the local changes, push to Github: - ``` - git push origin name-of-your-bugfix-or-feature - ``` - Note that each push will trigger the Continuous Integration workflow. +7. Add your name to the .zenodo.json file as an author + +8. Commit your changes: + ``` + git add . + git commit -m "AAA: Brief description of your changes" + ``` + Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). + `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), + see the discussion there for a full list of this shorthand notation. + +9. Once you are happy with the local changes, push to Github: + ``` + git push origin name-of-your-bugfix-or-feature + ``` + Note that each push will trigger the Continuous Integration workflow. -9. Submit a pull request through the GitHub website. Pull requests should be - made to the ``develop`` branch. +10. Submit a pull request through the GitHub website. Pull requests should be + made to the ``develop`` branch. Note that automated tests will be run on + github actions, but these must be initialized by a member of the pysat team. Pull Request Guidelines ----------------------- @@ -160,3 +171,5 @@ These include: * Block and inline comments should use proper English grammar and punctuation with the exception of single sentences in a block, which may then omit the final period +* When casting is necessary, use `np.int64` and `np.float64` to ensure operating + system agnosticism diff --git a/LICENSE b/LICENSE index c9a10064..666d8f5d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2020, pysat +Copyright (c) 2023, pysat All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index d04d1045..9c96b121 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
- pysatNASA + pysatNASA
# pysatNASA: pysat support for NASA Space Science instruments @@ -20,37 +20,41 @@ some examples on how to use the routines pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports -Python 3.8+. - -| Common modules | Community modules | -| ---------------- | ----------------- | -| beautifulsoup4 | cdflib | -| lxml | pysat>=3.0.4 | -| netCDF4 | | -| numpy | | -| pandas | | -| requests | | -| xarray<2022.11 | | +Python 3.6+. + +| Common modules | Community modules | Optional Modules | +| ---------------- | ----------------- |------------------| +| beautifulsoup4 | cdflib>=0.4.4 | pysatCDF | +| lxml | pysat>=3.1.0 | | +| netCDF4 | | | +| numpy | | | +| pandas | | | +| requests | | | +| xarray | | | + +## PyPi Installation +``` +pip install pysatNASA +``` ## GitHub Installation -Currently, the main way to get pysatNASA is through github. - ``` git clone https://github.com/pysat/pysatNASA.git ``` -Change directories into the repository folder and run the setup.py file. For +Change directories into the repository folder and build the project. For a local install use the "--user" flag after "install". ``` cd pysatNASA/ -python setup.py install +pip install . ``` Note: pre-1.0.0 version ----------------------- -pysatNASA is currently in an initial development phase and requires pysat 3.0.4. +pysatNASA is currently in an initial development phase and requires pysat 3.1.0. +Feedback and contributions are appreciated. # Using with pysat @@ -62,7 +66,9 @@ from pysatNASA.instruments import icon_ivm ivm = pysat.Instrument(inst_module=icon_ivm, inst_id='a') ``` -Another way to use the instruments in an external repository is to register the instruments. This only needs to be done the first time you load an instrument. Afterward, pysat will identify them using the `platform` and `name` keywords. +Another way to use the instruments in an external repository is to register the +instruments. This only needs to be done the first time you load an instrument. +Afterward, pysat will identify them using the `platform` and `name` keywords. ``` import pysat @@ -70,3 +76,19 @@ import pysat pysat.utils.registry.register(['pysatNASA.instruments.icon_ivm']) ivm = pysat.Instrument('icon', 'ivm', inst_id='a') ``` + +# CDF Integration +For data products stored as CDF files, this package can use either `cdflib` or +`pysatCDF`. Note that `cdflib` is a pure python package and more readily +deployable across systems, whereas `pysatCDF` interfaces with the fortran. +This is a faster approach for loading data, but may not install on all systems. +There are known issues with `numpy`>=1.24. Therefore, `pysatCDF` is optional +rather than required. + +You can specify which load routine to use via the optional `use_cdflib` kwarg. +If no kwarg is specified, `pysatNASA` will default to `pysatCDF` if it is +successfully installed, and default to `cdflib` otherwise. + +``` +ivm = pysat.Instrument('cnofs', 'ivm', use_cdflib=True) +``` diff --git a/docs/conf.py b/docs/conf.py index c3398628..43245c9d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,7 @@ import json import os +import pkg_resources import sys sys.path.insert(0, os.path.abspath('..')) @@ -52,19 +53,18 @@ master_doc = 'index' # General information about the project. -zenodo = json.loads(open('../.zenodo.json').read()) project = 'pysatNASA' -author = ', '.join([creator['name'] for creator in zenodo['creators']]) -manual_copyright = ', '.join(['2021', author]) title = '{:s} Documentation'.format(project) +zenodo = json.loads(open('../.zenodo.json').read()) +author = ', '.join([creator['name'] for creator in zenodo['creators']]) +manual_copyright = ', '.join(['2023', author]) +category = 'Space Physics' description = 'Tools for NASA CDAWeb instruments.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -doc_dir = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(doc_dir, "..", project, "version.txt"), "r") as fin: - version = fin.read().strip() +version = pkg_resources.get_distribution('pysatNASA').version release = '{:s}-alpha'.format(version) # Include alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation @@ -96,7 +96,7 @@ # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_logo = os.path.join(os.path.abspath('.'), 'figures', 'logo.png') +html_logo = os.path.join(os.path.abspath('.'), 'figures', 'pysatnasa_logo.png') html_theme_options = {'logo_only': True} # Add any paths that contain custom static files (such as style sheets) here, @@ -148,8 +148,8 @@ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [(master_doc, project, title, author, project, description, - 'Space Physics')] +texinfo_documents = [(master_doc, project, title, author, project, + description, category)] # -- Options for Epub output ---------------------------------------------- diff --git a/docs/figures/logo.png b/docs/figures/pysatnasa_logo.png similarity index 100% rename from docs/figures/logo.png rename to docs/figures/pysatnasa_logo.png diff --git a/docs/installation.rst b/docs/installation.rst index 82f840fa..7b5858b7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -14,18 +14,18 @@ Prerequisites pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports -Python 3.8+ and pysat 3.0.4+. +Python 3.6+ and pysat 3.1.0+. ================== ================= Common modules Community modules ================== ================= beautifulsoup4 cdflib>=0.4.4 - lxml pysat>=3.0.4 + lxml pysat>=3.1.0 netCDF4 numpy pandas requests - xarray<2022.11 + xarray ================== ================= @@ -40,24 +40,32 @@ Installation Options 2. Install pysatNASA: - Change directories into the repository folder and run the setup.py file. + Change directories into the repository folder and build the project. There are a few ways you can do this: A. Install on the system (root privileges required):: - sudo python setup.py install + sudo pip install . + B. Install at the user level:: - python setup.py install --user - C. Install with the intent to develop locally:: + pip install --user . + + C. Install with the intent to change the code:: + + + pip install --user -e . +.. extras-require:: pysatcdf + :pyproject: - python setup.py develop --user +.. extras-require:: test + :pyproject: -.. extras-require:: all - :setup.cfg: +.. extras-require:: doc + :pyproject: .. _post-install: diff --git a/docs/overview.rst b/docs/overview.rst index c103d568..4c8a8d4e 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -5,7 +5,7 @@ This is a library of ``pysat`` instrument modules and methods designed to suppor NASA instruments and missions archived at the Community Data Analysis Web portal. -.. image:: figures/logo.png +.. image:: figures/pysatnasa_logo.png :width: 400px :align: center :alt: pysatNASA Logo, a blue planet with red orbiting python and the module name superimposed diff --git a/docs/supported_constellations.rst b/docs/supported_constellations.rst index f5ebf71f..d7e7eaa3 100644 --- a/docs/supported_constellations.rst +++ b/docs/supported_constellations.rst @@ -6,10 +6,12 @@ DE2 The Dynamics Explorer 2 spacecraft. Includes the instruments +- :ref:`de2_fpi` - :ref:`de2_lang` - :ref:`de2_nacs` - :ref:`de2_rpa` - :ref:`de2_wats` +- :ref:`de2_vefi` .. automodule:: pysatNASA.constellations.de2 :members: diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 054bc982..0980414e 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -1,6 +1,38 @@ Supported Instruments ===================== +.. _ace_epam: + +ACE EPAM +-------- + +.. automodule:: pysatNASA.instruments.ace_epam_l2 + :members: + +.. _ace_mag: + +ACE MAG +------- + +.. automodule:: pysatNASA.instruments.ace_mag_l2 + :members: + +.. _ace_sis: + +ACE SIS +------- + +.. automodule:: pysatNASA.instruments.ace_sis_l2 + :members: + +.. _ace_swepam: + +ACE SWEPAM +---------- + +.. automodule:: pysatNASA.instruments.ace_swepam_l2 + :members: + .. _cnofs_ivm: C/NOFS IVM @@ -25,6 +57,14 @@ C/NOFS VEFI .. automodule:: pysatNASA.instruments.cnofs_vefi :members: +.. _de2_fpi: + +DE2 FPI +-------- + +.. automodule:: pysatNASA.instruments.de2_fpi + :members: + .. _de2_lang: DE2 LANG @@ -49,6 +89,14 @@ DE2 RPA .. automodule:: pysatNASA.instruments.de2_rpa :members: +.. _de2_vefi: + +DE2 VEFI +-------- + +.. automodule:: pysatNASA.instruments.de2_vefi + :members: + .. _de2_wats: DE2 WATS @@ -57,6 +105,14 @@ DE2 WATS .. automodule:: pysatNASA.instruments.de2_wats :members: +.. _dmsp_ssusi: + +DMSP SSUSI +---------- + +.. automodule:: pysatNASA.instruments.dmsp_ssusi + :members: + .. _formosat1_ivm: FORMOSAT-1 IVM @@ -98,6 +154,14 @@ ICON MIGHTI .. automodule:: pysatNASA.instruments.icon_mighti :members: +.. _igs_gps: + +IGS GPS +------- + +.. automodule:: pysatNASA.instruments.igs_gps + :members: + .. _iss_fpmu: ISS FPMU @@ -130,6 +194,14 @@ SES14 GOLD .. automodule:: pysatNASA.instruments.ses14_gold :members: +.. _timed_guvi: + +TIMED GUVI +---------- + +.. automodule:: pysatNASA.instruments.timed_guvi + :members: + .. _timed_saber: TIMED SABER diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..b35d7492 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,88 @@ +[build-system] +requires = ["setuptools >= 38.6", "pip >= 10"] +build-backend = "setuptools.build_meta" + +[project] +name = "pysatNASA" +version = "0.0.5" +description = "pysat support for NASA Instruments" +readme = "README.md" +requires-python = ">=3.6" +license = {file = "LICENSE"} +authors = [ + {name = "Jeff Klenzing, et al.", email = "pysat.developers@gmail.com"}, +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Topic :: Scientific/Engineering :: Astronomy", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows" +] +keywords = [ + "pysat", + "ionosphere", + "magnetosphere", + "solar wind", + "thermosphere" +] +dependencies = [ + "beautifulsoup4", + "cdasws", + "cdflib >= 0.4.4", + "lxml", + "netCDF4", + "numpy", + "pandas", + "pysat >= 3.1", + "requests", + "xarray" +] + +[project.optional-dependencies] +pysatcdf = ["pysatCDF"] +test = [ + "coveralls < 3.3", + "flake8", + "flake8-docstrings", + "hacking >= 1.0", + "pytest", + "pytest-cov", + "pytest-ordering" +] +doc = [ + "extras_require", + "ipython", + "m2r2", + "numpydoc", + "sphinx", + "sphinx_rtd_theme" +] + +[project.urls] +Documentation = "https://pysatnasa.readthedocs.io/en/latest/" +Source = "https://github.com/pysat/pysatNASA" + +[tool.coverage.report] +omit = ["*/instruments/templates/"] + +[tool.pytest.ini_options] +addopts = "-vs --cov=pysatNASA" +markers = [ + "all_inst", + "download", + "no_download", + "load_options", + "first", + "second" +] diff --git a/pysatNASA/__init__.py b/pysatNASA/__init__.py index 1df92789..ea9f8ff2 100644 --- a/pysatNASA/__init__.py +++ b/pysatNASA/__init__.py @@ -6,13 +6,15 @@ """ -import os +import importlib +import importlib_metadata + from pysatNASA import constellations # noqa F401 from pysatNASA import instruments # noqa F401 # set version -here = os.path.abspath(os.path.dirname(__file__)) -version_filename = os.path.join(here, 'version.txt') -with open(version_filename, 'r') as version_file: - __version__ = version_file.read().strip() -del here, version_filename, version_file +try: + __version__ = importlib.metadata.version('pysatNASA') +except AttributeError: + # Python 3.6 requires a different version + __version__ = importlib_metadata.version('pysatNASA') diff --git a/pysatNASA/constellations/de2.py b/pysatNASA/constellations/de2.py index eca99b6a..2a5ad572 100644 --- a/pysatNASA/constellations/de2.py +++ b/pysatNASA/constellations/de2.py @@ -1,13 +1,32 @@ -"""Creates a constellation from the NASA DE2 satellite platform.""" +"""Creates a constellation from the NASA DE2 satellite platform. + +Includes the core supported instruments. + + +Examples +-------- +:: + + import pysat + import pysatNASA + + de2 = pysat.Constellation(const_module=pysatNASA.constellations.de2) + + de2.load(1983, 1) + + +""" import pysat from pysatNASA import instruments +fpi = pysat.Instrument(inst_module=instruments.de2_fpi) lang = pysat.Instrument(inst_module=instruments.de2_lang) nacs = pysat.Instrument(inst_module=instruments.de2_nacs) rpa = pysat.Instrument(inst_module=instruments.de2_rpa) wats = pysat.Instrument(inst_module=instruments.de2_wats) +vefi = pysat.Instrument(inst_module=instruments.de2_vefi) -instruments = [lang, nacs, rpa, wats] +instruments = [fpi, lang, nacs, rpa, wats, vefi] diff --git a/pysatNASA/constellations/icon.py b/pysatNASA/constellations/icon.py index 3df198a0..42cd6462 100644 --- a/pysatNASA/constellations/icon.py +++ b/pysatNASA/constellations/icon.py @@ -1,4 +1,24 @@ -"""Creates a constellation from NASA the ICON satellite platform.""" +"""Creates a constellation from NASA the ICON satellite platform. + +Includes the core instruments without the line of sight winds. + +Note that IVM A and B are nominally never active at the same time. This +constellation should be initialized with `common_index=False`. This forgoes +the pysat check that ensures all instruments load data. + +Examples +-------- +:: + + import pysat + import pysatNASA + + icon = pysat.Constellation(const_module=pysatNASA.constellations.icon, + common_index=False) + + icon.load(2020, 1) + +""" import pysat diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index c3a8bfae..5407b031 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -4,13 +4,15 @@ Each instrument is contained within a subpackage of this set. """ +from pysatNASA.instruments import methods # noqa F401 -__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', - 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', - 'formosat1_ivm', +__all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2', + 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_fpi', + 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats', + 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', - 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', - 'timed_saber', 'timed_see'] + 'igs_gps', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', + 'timed_guvi', 'timed_saber', 'timed_see'] for inst in __all__: exec("from pysatNASA.instruments import {x}".format(x=inst)) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py new file mode 100644 index 00000000..61dddf8e --- /dev/null +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) EPAM instrument. + +Properties +---------- +platform + 'ace' +name + 'epam_l2' +tag + 'base' or 'key' +inst_id + '12sec', '5min', '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Gold, R., Krimigis, S., Hawkins, S. et al. Electron, Proton, and Alpha Monitor + on the Advanced Composition Explorer spacecraft. Space Science Reviews 86, + 541–562 (1998). https://doi.org/10.1023/A:1005088115759 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/epam/epam_level2_release_notes.txt + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'epam_l2' +tags = {'base': 'ACE/EPAM Solar Energetic Particle Base Data', + 'key': 'ACE/EPAM Solar Energetic Particle Key Parameters'} +inst_ids = {'12sec': ['base'], + '5min': ['key', 'base'], + '1hr': ['key', 'base']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# + +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'12sec': {'base': 'h3'}, + '5min': {'base': 'h1', 'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k1'}} +fname = ''.join(('ac_{sid:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(mm_ace.load, to_pandas=True) + +# Set the download routine +download_tags = {'12sec': {'base': 'AC_H3_EPM'}, + '5min': {'base': 'AC_H1_EPM', 'key': 'AC_K0_EPM'}, + '1hr': {'base': 'AC_H2_EPM', 'key': 'AC_K1_EPM'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py new file mode 100644 index 00000000..dfe26153 --- /dev/null +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) MAG instrument. + +Properties +---------- +platform + 'ace' +name + 'mag_l2' +tag + 'base' or 'key' +inst_id + '1sec', '16sec', '4min', '5min', '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Smith, C., L'Heureux, J., Ness, N. et al. The ACE Magnetic Fields Experiment. + Space Science Reviews 86, 613–632 (1998). + https://doi.org/10.1023/A:1005092216668 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/mag/mag_level2_release_notes.txt + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'mag_l2' +tags = {'base': 'ACE Magnetic Field Base Data', + 'key': 'ACE Magnetic Field Key Parameters'} +inst_ids = {'1sec': ['base'], + '16sec': ['base', 'key'], + '4min': ['base'], + '5min': ['key'], + '1hr': ['base', 'key']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + + +def preprocess(self): + """Adjust dimensionality of metadata.""" + + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing value_min0, value_min1, etc, once possible + self.meta['BGSEc'] = {'value_min': min([self.meta['BGSEc']['value_min0'], + self.meta['BGSEc']['value_min1'], + self.meta['BGSEc']['value_min2']]), + 'value_max': max([self.meta['BGSEc']['value_max0'], + self.meta['BGSEc']['value_max1'], + self.meta['BGSEc']['value_max2']]), + 'SCALEMIN': min([self.meta['BGSEc']['SCALEMIN0'], + self.meta['BGSEc']['SCALEMIN1'], + self.meta['BGSEc']['SCALEMIN2']]), + 'SCALEMAX': max([self.meta['BGSEc']['SCALEMAX0'], + self.meta['BGSEc']['SCALEMAX1'], + self.meta['BGSEc']['SCALEMAX2']])} + return + + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'1sec': {'base': 'h3'}, + '16sec': {'base': 'h0', 'key': 'k1'}, + '4min': {'base': 'h1'}, + '5min': {'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k2'}} +fname = ''.join(('ac_{sid:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(mm_ace.load, to_pandas=False) + +# Set the download routine +download_tags = {'1sec': {'base': 'AC_H3_MFI'}, + '16sec': {'base': 'AC_H0_MFI', 'key': 'AC_K1_MFI'}, + '4min': {'base': 'AC_H1_MFI'}, + '5min': {'key': 'AC_K0_MFI'}, + '1hr': {'base': 'AC_H2_MFI', 'key': 'AC_K2_MFI'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py new file mode 100644 index 00000000..188f5973 --- /dev/null +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) SIS instrument. + +Properties +---------- +platform + 'ace' +name + 'sis_l2' +tag + 'base' or 'key' +inst_id + '256sec' or '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Stone, E., Cohen, C., Cook, W. et al. The Solar Isotope Spectrometer for the + Advanced Composition Explorer. Space Science Reviews 86, 357–408 (1998). + https://doi.org/10.1023/A:1005027929871 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'sis_l2' +tags = {'base': 'ACE/SIS Solar Isotope Spectrometer Base Data', + 'key': 'ACE/SIS Solar Isotope Spectrometer Key Parameters'} +inst_ids = {'256sec': ['base'], + '1hr': ['base', 'key']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'256sec': {'base': 'h1'}, + '1hr': {'base': 'h2', 'key': 'k0'}} +fname = ''.join(('ac_{sid:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(mm_ace.load, to_pandas=False) + +# Set the download routine +download_tags = {'256sec': {'base': 'AC_H1_SIS'}, + '1hr': {'base': 'AC_H2_SIS', 'key': 'AC_K0_SIS'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py new file mode 100644 index 00000000..b16b37ed --- /dev/null +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) EPAM instrument. + +Properties +---------- +platform + 'ace' +name + 'swepam_l2' +tag + 'base' or 'key' +inst_id + '64sec', '5min', '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- McComas, D., Bame, S., Barker, P. et al. Solar Wind Electron Proton Alpha + Monitor (SWEPAM) for the Advanced Composition Explorer. Space Science Reviews + 86, 563–612 (1998). https://doi.org/10.1023/A:1005040232597 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/swepam/swepam_level2_release_notes.txt + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'swepam_l2' +tags = {'base': 'ACE/SWEPAM Solar Wind Experiment Base Data', + 'key': 'ACE/SWEPAM Solar Wind Experiment Key Parameters'} +inst_ids = {'64sec': ['base'], + '5min': ['key'], + '1hr': ['key', 'base']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2021, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'64sec': {'base': 'h0'}, + '5min': {'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k1'}} +fname = ''.join(('ac_{sid:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + + +# Set the load routine +load = functools.partial(mm_ace.load, to_pandas=True) + +# Set the download routine +download_tags = {'64sec': {'base': 'AC_H0_SWE'}, + '5min': {'key': 'AC_K0_SWE'}, + '1hr': {'base': 'AC_H2_SWE', 'key': 'AC_K1_SWE'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_ivm.py b/pysatNASA/instruments/cnofs_ivm.py index a7693648..59dbf57b 100644 --- a/pysatNASA/instruments/cnofs_ivm.py +++ b/pysatNASA/instruments/cnofs_ivm.py @@ -16,23 +16,6 @@ motion of the satellite the angle is converted into ion motion along two orthogonal directions, perpendicular to the satellite track. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. - -Discussion of cleaning parameters for ion drifts can be found in: -Burrell, Angeline G., Equatorial topside magnetic field-aligned ion drifts -at solar minimum, The University of Texas at Dallas, ProQuest -Dissertations Publishing, 2012. 3507604. - -Discussion of cleaning parameters for ion temperature can be found in: -Hairston, M. R., W. R. Coley, and R. A. Heelis (2010), Mapping the -duskside topside ionosphere with CINDI and DMSP, J. Geophys. Res.,115, -A08324, doi:10.1029/2009JA015051. - Properties ---------- @@ -45,6 +28,7 @@ inst_id None supported + Warnings -------- - The sampling rate of the instrument changes on July 29th, 2010. @@ -52,6 +36,24 @@ - The cleaning parameters for the instrument are still under development. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + +Discussion of cleaning parameters for ion drifts can be found in: +Burrell, Angeline G., Equatorial topside magnetic field-aligned ion drifts +at solar minimum, The University of Texas at Dallas, ProQuest +Dissertations Publishing, 2012. 3507604. + +Discussion of cleaning parameters for ion temperature can be found in: +Hairston, M. R., W. R. Coley, and R. A. Heelis (2010), Mapping the +duskside topside ionosphere with CINDI and DMSP, J. Geophys. Res.,115, +A08324, doi:10.1029/2009JA015051. + """ import datetime as dt @@ -245,12 +247,9 @@ def clean(self): # Set the load routine load = cdw.load -# Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/cindi/ivm_500ms_cdf/{year:4d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py index fb43ab9e..1b5b546e 100644 --- a/pysatNASA/instruments/cnofs_plp.py +++ b/pysatNASA/instruments/cnofs_plp.py @@ -29,12 +29,6 @@ The data is PRELIMINARY, and as such, is intended for BROWSE PURPOSES ONLY. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. Properties ---------- @@ -54,11 +48,18 @@ - Currently no cleaning routine. - Module not written by PLP team. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + """ import datetime as dt import functools -import numpy as np from pysat.instruments.methods import general as mm_gen @@ -86,21 +87,8 @@ init = functools.partial(mm_nasa.init, module=mm_cnofs, name=name) -def clean(self): - """Clean C/NOFS PLP data to the specified level. - - Note - ---- - Basic cleaning to find valid Epoch values - - """ - - for key in self.data.columns: - if key != 'Epoch': - fill = self.meta[key, self.meta.labels.fill_val] - idx, = np.where(self[key] == fill) - self[idx, key] = np.nan - return +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- @@ -119,11 +107,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/plp/plasma_1sec/{year:4d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'CNOFS_PLP_PLASMA_1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 78d54e36..f2e4bf24 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -26,12 +26,6 @@ The data is PRELIMINARY, and as such, is intended for BROWSE PURPOSES ONLY. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. Properties ---------- @@ -56,6 +50,14 @@ - Limited cleaning routine. - Module not written by VEFI team. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + """ import datetime as dt @@ -122,11 +124,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/vefi/bfield_1sec/{year:4d}/', - 'fname': fname} -download_tags = {'': {'dc_b': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_fpi.py b/pysatNASA/instruments/de2_fpi.py new file mode 100644 index 00000000..d27693ed --- /dev/null +++ b/pysatNASA/instruments/de2_fpi.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +"""The DE2 FPI instrument. + +Supports the Fabry-Perot Interferometer (FPI) instrument on Dynamics Explorer 2 +(DE2). + +From CDAWeb: + +The Fabry-Perot Interferometer (FPI) was a high-resolution remote sensing +instrument designed to measure the thermospheric temperature, meridional wind, +and density of the following metastable atoms: atomic oxygen (singlet S and D) +and the 2P state of ionic atomic oxygen. The FPI performed a wavelength analysis +on the light detected from the thermospheric emission features by spatially +scanning the interference fringe plane with a multichannel array detector. The +wavelength analysis characterized the Doppler line profile of the emitting +species. A sequential altitude scan performed by a commandable horizon scan +mirror provided a cross-sectional view of the thermodynamic and dynamic state of +the thermosphere below the DE 2 orbit. The information obtained from this +investigation was used to study the dynamic response of the thermosphere to the +energy sources caused by magnetospheric electric fields and the absorption of +solar ultraviolet light in the thermosphere. The instrument was based on the +visible airglow experiment (VAE) used in the AE program. The addition of a +scanning mirror, the Fabry-Perot etalon, an image plane detector, and a +calibration lamp were the principal differences. Interference filters isolated +lines at (in Angstroms) 5577, 6300, 7320, 5896, and 5200. The FPI had a field of +view of 0.53 deg (half-cone angle). More details are found in P. B. Hays et al., +Space Sci. Instrum., v. 5, n. 4, p. 395, 1981. From February 16, 1982 to +September 11, 1982 the DE satellite was inverted and the FPI measured galactic +emissions. + +Properties +---------- +platform + 'de2' +name + 'fpi' +tag + None Supported +inst_id + None Supported + + +Warnings +-------- +- Currently no cleaning routine. + + +References +---------- +Hays, P B, Killeen, T L, and Kennedy, B C. "Fabry-Perot interferometer on +Dynamics Explorer". Space Sci. Instrum., 5, 395-416, 1981. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'fpi' +tags = {'': '8 s cadence Fabry-Perot Interferometer data'} +inst_ids = {'': [tag for tag in tags.keys()]} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = 'de2_neutral8s_fpi_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Use the default CDAWeb method +load = cdw.load + +# Support download routine +download_tags = {'': {'': 'DE2_NEUTRAL8S_FPI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Support listing files currently on CDAWeb +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index 07430357..28d4dbc9 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -25,22 +25,16 @@ or correct the inflight processed data. Time resolution was 0.5 seconds. -References ----------- -J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, -"The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, -493-502, 1981. - Properties ---------- platform 'de2' name 'lang' -inst_id - None Supported tag None Supported +inst_id + None Supported Warnings @@ -48,6 +42,12 @@ - Currently no cleaning routine. +References +---------- +J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, +"The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, +493-502, 1981. + """ import datetime as dt @@ -65,12 +65,12 @@ platform = 'de2' name = 'lang' tags = {'': '500 ms cadence Langmuir Probe data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -79,8 +79,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions @@ -97,12 +97,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_lang', - '/plasma500ms_lang_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_PLASMA500MS_LANG'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index b03212fe..ba949e95 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -50,27 +50,29 @@ were lost between 12 March 1982 and 31 March 1982 when the counter overflowed. -References ----------- -G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, -N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", -Space Sci. Instrum., 5, 429-441, 1981. - Properties ---------- platform 'de2' name 'nacs' -inst_id - None Supported tag None Supported +inst_id + None Supported + Warnings -------- - Currently no cleaning routine. + +References +---------- +G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, +N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", +Space Sci. Instrum., 5, 429-441, 1981. + """ import datetime as dt @@ -88,12 +90,12 @@ platform = 'de2' name = 'nacs' tags = {'': '1 s cadence Neutral Atmosphere Composition Spectrometer data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -102,8 +104,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions @@ -120,12 +122,9 @@ load = cdw.load # Support download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_nacs', - '/neutral1s_nacs_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_NEUTRAL1S_NACS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Support listing files currently on CDAWeb -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 912dd66f..aa97b66d 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -26,21 +26,11 @@ spectrum; and the concentration of H+, He+, O+, and Fe+, and of molecular ions near perigee. -It includes the DUCT portion of the high resolutiondata from the Dynamics -Explorer 2 (DE-2) Retarding Potential Analyzer (RPA) for the whole DE-2 mission -time period in ASCII format. This version was generated at NSSDC from the -PI-provided binary data (SPIO-00232). The DUCT files include RPA measurements -ofthe total ion concentration every 64 times per second. Due to a failure in -the instrument memory system RPA data are not available from 81317 06:26:40 UT -to 82057 13:16:00 UT. This data set is based on the revised version of the RPA -files that was submitted by the PI team in June of 1995. The revised RPA data -include a correction to the spacecraft potential. +Due to a failure in the instrument memory system RPA data are not available +from 81317 06:26:40 UT to 82057 13:16:00 UT. This data set is based on the +revised version of the RPA files that was submitted by the PI team in June of +1995. The revised RPA data include a correction to the spacecraft potential. -References ----------- -W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, -B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer -for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). Properties ---------- @@ -48,15 +38,23 @@ 'de2' name 'rpa' -inst_id - None Supported tag None Supported +inst_id + None Supported + Warnings -------- - Currently no cleaning routine. + +References +---------- +W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, +B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer +for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). + """ import datetime as dt @@ -73,13 +71,13 @@ platform = 'de2' name = 'rpa' -tags = {'': '2 sec cadence RPA data'} # this is the default cadence -inst_ids = {'': ['']} +tags = {'': '2 sec cadence RPA data'} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -88,8 +86,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions @@ -97,8 +95,11 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = 'de2_ion2s_rpa_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' -supported_tags = {'': {'': fname}} +datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' +dataproduct = {'': 'ion2s'} +fname = 'de2_{dp:s}_rpa_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(dp=dataproduct[tag], datestr=datestr) + for tag in tags.keys()}} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -106,12 +107,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_rpa', - '/ion2s_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_ION2S_RPA'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py new file mode 100644 index 00000000..aba68448 --- /dev/null +++ b/pysatNASA/instruments/de2_vefi.py @@ -0,0 +1,105 @@ +"""Module for the DE2 VEFI instrument. + +From CDAWeb (adpated): +This directory gathers data for the VEFI instrument that flew on the DE 2 +spacecraft which was launched on 3 August 1981 into an elliptical orbit with +an altitude range of 300 km to 1000 km and re-entered the atmosphere on +19 February 1983. + +dca (NSSDC ID: 81-070B-02C) + +This data set contains the averaged (2 samples per second) DC electric fields in +spacecraft coordinates and orbit information in ASCII format. + +ac (NSSDC ID: 81-070B-02E) + +This data set contains the averaged AC electric field data (1 or 2 points per +second) and orbit information. + +References +---------- +Maynard, N. C., E. A. Bielecki, H. G. Burdick, Instrumentation for vector +electric field measurements from DE-B, Space Sci. Instrum., 5, 523, 1981. + +Properties +---------- +platform + 'de2' +name + 'vefi' +tag + 'dca' or 'ac' +inst_id + None Supported + + +Warnings +-------- +- Currently no cleaning routine. + + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'vefi' +tags = {'': '62 ms combination of Electric Field and Magnetometer', + 'dca': '500 ms cadence DC Averaged Electric Field data', + 'ac': '500 ms cadence AC Electric Field data'} +inst_ids = {'': [tag for tag in tags.keys()]} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} + + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' +fid = {'': '62ms_vefimagb', + 'ac': 'ac500ms_vefi', + 'dca': 'dca500ms_vefi'} +fname = 'de2_{fid:s}_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(fid=fid[tag], datestr=datestr) + for tag in tags.keys()}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +# Forcing use of cdflib as default since pysatCDF has a known issue with vefi +# data. See pysat/pysatCDF#48 +load = functools.partial(cdw.load, use_cdflib=True) + +# Set the download routine +download_tags = {'': {'': 'DE2_62MS_VEFIMAGB', + 'ac': 'DE2_AC500MS_VEFI', + 'dca': 'DE2_DCA500MS_VEFI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index 1345841a..5a784d24 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -47,27 +47,29 @@ about the processing done at NSSDC is given in WATS_NSSDC_PRO_DE.DOC. -References ----------- -N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, -J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", -Space Sci. Instrum., 5, 417-428, 1981. - Properties ---------- platform 'de2' name 'wats' -inst_id - None Supported tag None Supported +inst_id + None Supported + Warnings -------- - Currently no cleaning routine. + +References +---------- +N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, +J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", +Space Sci. Instrum., 5, 417-428, 1981. + """ import datetime as dt @@ -85,12 +87,12 @@ platform = 'de2' name = 'wats' tags = {'': '2 s cadence Wind and Temperature Spectrometer data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -99,8 +101,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions @@ -117,12 +119,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_wats', - '/wind2s_wats_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_WIND2S_WATS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py new file mode 100644 index 00000000..268985a8 --- /dev/null +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +"""Module for the DMSP SSUSI instrument. + +Supports the Special Sensor Ultraviolet Spectrographic Imager (SSUSI) +instrument on Defense Meteorological Satellite Program (DMSP). + +From JHU APL: + +SSUSI was designed for the DMSP Block 5D-3 satellites. These satellites are +placed into nearly polar, sun-synchronous orbits at an altitude of about 850 km. +SSUSI is a remote-sensing instrument which measures ultraviolet (UV) emissions +in five different wavelength bands from the Earth's upper atmosphere. SSUSI is +mounted on a nadir-looking panel of the satellite. The multicolor images from +SSUSI cover the visible Earth disk from horizon to horizon and the anti-sunward +limb up to an altitude of approximately 520 km. + +The UV images and the derived environmental data provide the Air Force Weather +Agency (Offutt Air Force Base, Bellevue, NE) with near real-time information +that can be utilized in a number of applications, such as maintenance of high +frequency (HF) communication links and related systems and assessment of the +environmental hazard to astronauts on the Space Station. + + +Properties +---------- +platform + 'dmsp' +name + 'ssusi' +tag + 'edr-aurora' +inst_id + 'f16', 'f17', 'f18', 'f19' + + +Warnings +-------- +- Currently no cleaning routine. + + +References +---------- +Larry J. Paxton, Daniel Morrison, Yongliang Zhang, Hyosub Kil, Brian Wolven, +Bernard S. Ogorzalek, David C. Humm, and Ching-I. Meng "Validation of remote +sensing products produced by the Special Sensor Ultraviolet Scanning Imager +(SSUSI): a far UV-imaging spectrograph on DMSP F-16", Proc. SPIE 4485, Optical +Spectroscopic Techniques, Remote Sensing, and Instrumentation for Atmospheric +and Space Research IV, (30 January 2002); doi:10.1117/12.454268 + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import dmsp as mm_dmsp +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import jhuapl + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'dmsp' +name = 'ssusi' +tags = {'edr-aurora': ''.join(['Electron energy flux and mean energy, auroral', + ' boundaries, identified discrete auroral arcs,', + ' hemispheric power, and magnetic field lines ', + 'traced to 4 Earth radii'])} +inst_ids = {sat_id: list(tags.keys()) + for sat_id in ['f16', 'f17', 'f18', 'f19']} + +pandas_format = False +multi_file_day = True + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: dt.datetime(2015, 1, 1) for tag in tags.keys()} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_dmsp, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(['dmsp{inst_id:s}_ssusi_{tag:s}_{{year:04d}}{{day:03d}}T', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-???????T??????-REV', + '?????_vA{{version:1d}}.?.?r{{cycle:03d}}.nc']) +supported_tags = {sat_id: {tag: fname.format(tag=tag, inst_id=sat_id) + for tag in tags.keys()} + for sat_id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format, + strict_dim_check=False) + +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', + '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')), + 'fname': fname} +download_tags = { + sat_id: {tag: {btag: basic_tag[btag].format(tag=tag, inst_id=sat_id) + for btag in basic_tag.keys()} for tag in tags.keys()} + for sat_id in inst_ids.keys()} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 24be65bd..3ac177d9 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Module for the ICON EUV instrument. +"""Module for the Formosat-1 IVM instrument. Supports the Ion Velocity Meter (IVM) onboard the Formosat-1 (formerly ROCSAT-1) mission. Downloads data from the NASA Coordinated Data Analysis @@ -74,8 +74,8 @@ def init(self): return -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions @@ -92,12 +92,9 @@ def init(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/formosat-rocsat/formosat-1', - '/ipei/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'RS_K0_IPEI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index 7a5be9e4..a3658154 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -13,6 +13,8 @@ 'euv' tag None supported +inst_id + None Supported Warnings -------- @@ -127,13 +129,11 @@ def clean(self): supported_tags=supported_tags) # Set the download routine -basic_tag = {'remote_dir': '/pub/data/icon/l2/l2-6_euv/{year:04d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'ICON_L2-6_EUV'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -214,19 +214,21 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): """ labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot'} data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', pandas_format=pandas_format, - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=['Valid_Max', 'Valid_Min', - '_FillValue']) + '_FillValue'], + decode_times=False) # xarray can't merge if variable and dim names are the same if 'Altitude' in data.dims: diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 19b3cae5..38515810 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -13,6 +13,8 @@ 'fuv' tag None supported +inst_id + None Supported Warnings -------- @@ -119,16 +121,13 @@ def clean(self): supported_tags=supported_tags) # Set the download routine -basic_tag24 = {'remote_dir': '/pub/data/icon/l2/l2-4_fuv_day/{year:04d}/', - 'fname': fname24} -basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/', - 'fname': fname25} -download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}} +download_tags = {'': {'day': 'ICON_L2-4_FUV_DAY', + 'night': 'ICON_L2-5_FUV_NIGHT'}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -211,8 +210,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): """ labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis', 'FIELDNAM': 'plot', 'LABLAXIS': 'axis', @@ -223,8 +223,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', pandas_format=pandas_format, - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, - drop_meta_labels=drop_labels) + drop_meta_labels=drop_labels, + decode_times=False) return data, meta diff --git a/pysatNASA/instruments/icon_ivm.py b/pysatNASA/instruments/icon_ivm.py index e16ca5b0..ff6012d0 100644 --- a/pysatNASA/instruments/icon_ivm.py +++ b/pysatNASA/instruments/icon_ivm.py @@ -192,13 +192,12 @@ def clean(self): # Set the download routine dirstr = '/pub/data/icon/l2/l2-7_ivm-{id:s}/{{year:4d}}/' -download_tags = {id: {'': {'remote_dir': dirstr.format(id=id), - 'fname': supported_tags[id]['']}} - for id in ['a', 'b']} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'a': {'': 'ICON_L2-7_IVM-A'}, 'b': {'': 'ICON_L2-7_IVM-B'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -273,15 +272,16 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis', 'ScaleTyp': 'scale', '_FillValue': 'FillVal'} data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=['Valid_Max', diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py index 006a9f8a..a108e8d2 100644 --- a/pysatNASA/instruments/icon_mighti.py +++ b/pysatNASA/instruments/icon_mighti.py @@ -226,30 +226,23 @@ def _clean_vars(var_list, flag, min_level): supported_tags=supported_tags) # Set the download routine -dirstr1 = '/pub/data/icon/l2/l2-1_mighti-{{id:s}}_los-wind-{color:s}/' -dirstr2 = '/pub/data/icon/l2/l2-2_mighti_vector-wind-{color:s}/' -dirstr3 = '/pub/data/icon/l2/l2-3_mighti-{id:s}_temperature/' -dirnames = {'los_wind_green': dirstr1.format(color='green'), - 'los_wind_red': dirstr1.format(color='red'), - 'vector_wind_green': dirstr2.format(color='green'), - 'vector_wind_red': dirstr2.format(color='red'), - 'temperature': dirstr3} - -download_tags = {} -for inst_id in supported_tags.keys(): - download_tags[inst_id] = {} - for tag in supported_tags[inst_id].keys(): - fname = supported_tags[inst_id][tag] - - download_tags[inst_id][tag] = { - 'remote_dir': ''.join((dirnames[tag].format(id=inst_id), - '{year:04d}/')), - 'fname': fname} - -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'vector': + {'vector_wind_green': 'ICON_L2-2_MIGHTI_VECTOR-WIND-GREEN', + 'vector_wind_red': 'ICON_L2-2_MIGHTI_VECTOR-WIND-RED'}, + 'a': + {'los_wind_green': 'ICON_L2-1_MIGHTI-A_LOS-WIND-GREEN', + 'los_wind_red': 'ICON_L2-1_MIGHTI-A_LOS-WIND-RED', + 'temperature': 'ICON_L2-3_MIGHTI-A_TEMPERATURE'}, + 'b': + {'los_wind_green': 'ICON_L2-1_MIGHTI-B_LOS-WIND-GREEN', + 'los_wind_red': 'ICON_L2-1_MIGHTI-B_LOS-WIND-RED', + 'temperature': 'ICON_L2-3_MIGHTI-B_TEMPERATURE'}} + + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -325,8 +318,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): """ labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis', 'FIELDNAM': 'plot', 'LABLAXIS': 'axis', @@ -335,14 +329,15 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', pandas_format=pandas_format, - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=['Valid_Max', 'Valid_Min', 'Valid_Range', '_Fillvalue', - 'ScaleTyp']) + 'ScaleTyp'], + decode_times=False) # xarray can't merge if variable and dim names are the same if 'Altitude' in data.dims: diff --git a/pysatNASA/instruments/igs_gps.py b/pysatNASA/instruments/igs_gps.py new file mode 100644 index 00000000..99cd4690 --- /dev/null +++ b/pysatNASA/instruments/igs_gps.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +"""Module for the IGS GPS data products. + +Supports GPS data produced from International GNSS Service Total Electron +Content (TEC). + +From CDAWeb (modified): + +This directory contains the GPS Total Electron Content (TEC) data produced by +the International Global Navigation Satellite Systems Service (IGS) Ionosphere +Working Group and by the Analysis Centers that have contributed to the IGS data +including CODE of the University of Bern (Switzerland), ESA of the European +Space Operations Center (ESOC) in Darmstadt (Germany), JPL of the Jet Propulsion +Laboratory, Pasadena (USA), and UPC of the University Politechnical Catalonia in +Barcelona (Spain). The IGS data are a computed as a weighted mean of the data +from the four analysis centers. + +The rate of TEC index (ROTI) characterizes TEC fluctuations observed along +receiver-to-satellite line of sight links over a 5-minute interval. +The measurement is obtained by processing GNSS dual-frequency phase data and +computing the standard deviation of the rate of TEC change over that interval +after removing its background variation trend. + +ROTI data are provided as global maps using a 2.5 x 5 degree (geographic +latitude x longitude) grid. The median ROTI value is calculated in each bin. +GNSS data contributing to the ROTI computation are primarily collected from +the global network of International GNSS Service and the regional network of +Continuous Operating Reference Station (CORS). + +Properties +---------- +platform + 'igs' +name + 'gps' +tag + ['tec', 'roti'] +inst_id + ['15min', '1hr', '2hr'] + + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + + +References +---------- +M. Hernández-Pajares, J.M. Juan, J. Sanz, R. Orus, A. Garcia-Rigo, J. Feltens, +A. Komjathy, S.C. Schaer, and A. Krankowski, The IGS VTEC maps: a reliable +source of ionospheric information since 1998Journal of Geodesy (2009) 83:263–275 +doi:10.1007/s00190-008-0266-1 + +Feltens, J., M. Angling, N. Jackson‐Booth, N. Jakowski, M. Hoque, M. +Hernández‐Pajares, A. Aragón‐Àngel, R. Orús, and R. Zandbergen (2011), +Comparative testing of four ionospheric models driven with GPS measurements, +Radio Sci., 46, RS0D12, doi:10.1029/2010RS004584 + +Peng Chen, Hang Liu, Yongchao Ma, Naiquan Zheng, Accuracy and consistency of +different global ionospheric maps released by IGS ionosphere associate analysis +centers, Advances in Space Research, Volume 65, Issue 1, 2020, Pages 163-174, +doi:10.1016/j.asr.2019.09.042. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import igs as mm_igs + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'igs' +name = 'gps' +tags = {'tec': 'Total Electron Content', + 'roti': 'Rate of Change in TEC'} +# tags = {'15min': '15 min cadence TEC', +# '1hr': '1 hour cadence TEC', +# '2hr': '2 hour cadence TEC'} +inst_ids = {'15min': ['tec', 'roti'], + '1hr': ['tec'], + '2hr': ['tec']} +pandas_format = False +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {jj: {kk: dt.datetime(2013, 1, 1) for kk in inst_ids[jj]} + for jj in inst_ids.keys()} +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_igs, name=name) + + +# Use default clean +clean = mm_nasa.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +cdas_labels = {'15min': {'tec': 'GPS_TEC15MIN_IGS', + 'roti': 'GPS_ROTI15MIN_JPL'}, + '1hr': {'tec': 'GPS_TEC1HR_IGS'}, + '2hr': {'tec': 'GPS_TEC2HR_IGS'}} + +date_ver = '{year:4d}{month:02d}{day:02d}_v{version:02d}' +fname = '{cdas:s}_{date_ver:s}.cdf' + +supported_tags = {id: {tag: fname.format(cdas=cdas_labels[id][tag].lower(), + date_ver=date_ver) + for tag in inst_ids[id]} for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(cdw.load, pandas_format=pandas_format, + use_cdflib=True) + +# Set the download routine +download = functools.partial(cdw.cdas_download, supported_tags=cdas_labels) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=cdas_labels) diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py index b85c6aa5..c98f907c 100644 --- a/pysatNASA/instruments/iss_fpmu.py +++ b/pysatNASA/instruments/iss_fpmu.py @@ -26,12 +26,12 @@ import datetime as dt import functools -import numpy as np from pysat.instruments.methods import general as mm_gen from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- # Instrument attributes @@ -79,25 +79,8 @@ def init(self): return -def clean(self): - """Clean ISS FPMU data to the specified level. - - Note - ---- - 'clean' - Replace Te and Ni fill values with NaN - 'dusty' - Same as clean - 'dirty' - Same as clean - 'none' - Not applied, default fill values are preserved - - """ - - # Replace Te data fill - self.data.replace(-999., np.nan, inplace=True) - - # Replace Ne data fill - self.data.replace(-9.9999998e+30, np.nan, inplace=True) - - return +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- @@ -115,12 +98,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/international_space_station_iss', - '/sp_fpmu/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'ISS_SP_FPMU'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index ddbe66cf..040ceac0 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- """Module for the JPL GPS data products. +.. deprecated:: 0.0.5 + This module is now included in igs_gps.py. + This instrument will be removed in 0.1.0+ to reduce redundancy. + + Supports ROTI data produced at JPL from International GNSS Service Total Electron Content (TEC) @@ -16,16 +21,6 @@ the global network of International GNSS Service and the regional network of Continuous Operating Reference Station (CORS). -References ----------- -Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global -ionospheric irregularities using the worldwide GPS network, Geophys. Res. -Lett., 24, 2283, 1997. - -Pi, X., F. J. Meyer, K. Chotoo, Anthony Freeman, R. G. Caton, and C. T. -Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations -studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. - Properties ---------- @@ -38,14 +33,27 @@ inst_id None supported + Warnings -------- - The cleaning parameters for the instrument are still under development. + +References +---------- +Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global +ionospheric irregularities using the worldwide GPS network, Geophys. Res. +Lett., 24, 2283, 1997. + +Pi, X., F. J. Meyer, K. Chotoo, Anthony Freeman, R. G. Caton, and C. T. +Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations +studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. + """ import datetime as dt import functools +import warnings import pysat from pysat.instruments.methods import general as mm_gen @@ -78,6 +86,10 @@ def init(self): """ + warnings.warn(" ".join(["The instrument module `jpl_gps` has", + "been deprecated and will be removed in 0.1.0+."]), + DeprecationWarning, stacklevel=2) + pysat.logger.info('') self.acknowledgements = mm_gps.ackn_str self.references = '\n'.join((mm_gps.refs['mission'], @@ -86,8 +98,8 @@ def init(self): return -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions @@ -101,14 +113,13 @@ def init(self): supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +load = functools.partial(cdw.load, pandas_format=pandas_format, + use_cdflib=True) # Set the download routine -basic_tag = {'remote_dir': '/pub/data/gps/roti15min_jpl/{year:4d}/', - 'fname': fname} -download_tags = {'': {'roti': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'roti': 'GPS_ROTI15MIN_JPL'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 5bde9a41..8542fcbc 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -4,6 +4,11 @@ from pysatNASA.instruments.methods import cdaweb # noqa F401 from pysatNASA.instruments.methods import cnofs # noqa F401 from pysatNASA.instruments.methods import de2 # noqa F401 +from pysatNASA.instruments.methods import dmsp # noqa F401 from pysatNASA.instruments.methods import general # noqa F401 +from pysatNASA.instruments.methods import gps # noqa F401 from pysatNASA.instruments.methods import icon # noqa F401 +from pysatNASA.instruments.methods import jhuapl # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 +from pysatNASA.instruments.methods import ses14 # noqa F401 +from pysatNASA.instruments.methods import timed # noqa F401 diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index 29071ff8..c02dc55d 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -87,12 +87,18 @@ def __init__(self, filename, self._cdf_file = cdflib.CDF(self._filename) self._cdf_info = self._cdf_file.cdf_info() + self.global_attrs = self._cdf_file.globalattsget() self.data = {} self.meta = {} self._dependencies = {} - self._variable_names = (self._cdf_info['rVariables'] - + self._cdf_info['zVariables']) + if hasattr(self._cdf_info, 'rVariables'): + self._variable_names = (self._cdf_info.rVariables + + self._cdf_info.zVariables) + else: + # cdflib < 1.0 stores info as a dict + self._variable_names = (self._cdf_info['rVariables'] + + self._cdf_info['zVariables']) self.load_variables() @@ -156,8 +162,13 @@ def set_epoch(self, x_axis_var): """ - data_type_description = self._cdf_file.varinq( - x_axis_var)['Data_Type_Description'] + if hasattr(self._cdf_file.varinq(x_axis_var), 'Data_Type_Description'): + data_type_description = self._cdf_file.varinq( + x_axis_var).Data_Type_Description + else: + # cdflib < 1.0 stores this as a dict + data_type_description = self._cdf_file.varinq( + x_axis_var)['Data_Type_Description'] center_measurement = self._center_measurement cdf_file = self._cdf_file @@ -298,7 +309,12 @@ def load_variables(self): if not re.match(var_regex, variable_name): # Skip this variable continue - var_atts = self._cdf_file.varattsget(variable_name, to_np=True) + try: + var_atts = self._cdf_file.varattsget(variable_name, to_np=True) + except TypeError: + # cdflib 1.0+ drops to_np kwarg, assumes True + var_atts = self._cdf_file.varattsget(variable_name) + for k in var_atts: var_atts[k] = var_atts[k] # [0] @@ -319,13 +335,14 @@ def load_variables(self): continue if "FILLVAL" in var_atts: - if (var_properties['Data_Type_Description'] == 'CDF_FLOAT' - or var_properties['Data_Type_Description'] - == 'CDF_REAL4' - or var_properties['Data_Type_Description'] - == 'CDF_DOUBLE' - or var_properties['Data_Type_Description'] - == 'CDF_REAL8'): + if hasattr(var_properties, 'Data_Type_Description'): + data_type_desc = var_properties.Data_Type_Description + else: + # cdflib < 1.0 stores this as a dict + data_type_desc = var_properties['Data_Type_Description'] + + if data_type_desc in ['CDF_FLOAT', 'CDF_REAL4', 'CDF_DOUBLE', + 'CDF_REAL8']: if ydata[ydata == var_atts["FILLVAL"]].size != 0: ydata[ydata == var_atts["FILLVAL"]] = np.nan @@ -360,9 +377,9 @@ def load_variables(self): def to_pysat(self, flatten_twod=True, labels={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), - 'fill_val': ('FillVal', float)}): + 'min_val': ('ValidMin', (float, int, str)), + 'max_val': ('ValidMax', (float, int, str)), + 'fill_val': ('FillVal', (float, int, str))}): """Export loaded CDF data into data, meta for pysat module. Parameters @@ -382,9 +399,9 @@ def to_pysat(self, flatten_twod=True, that order. (default={'units': ('units', str), 'name': ('long_name', str), 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', float), - 'max_val': ('value_max', float) - 'fill_val': ('fill', float)}) + 'min_val': ('value_min', (float, int, str)), + 'max_val': ('value_max', (float, int, str)) + 'fill_val': ('fill', (float, int, str))}) Returns ------- @@ -407,7 +424,7 @@ def to_pysat(self, flatten_twod=True, # and utilizing the attribute labels provided by the user meta = pysat.Meta(pds.DataFrame.from_dict(self.meta, orient='index'), labels=labels) - + meta.header = pysat.MetaHeader(header_data=self.global_attrs) cdata = self.data.copy() lower_names = [name.lower() for name in meta.keys()] for name, true_name in zip(lower_names, meta.keys()): diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py new file mode 100644 index 00000000..f9e78412 --- /dev/null +++ b/pysatNASA/instruments/methods/ace.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for ACE data.""" + +from pysatNASA.instruments.methods import cdaweb as cdw + +ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", + "Center, the Space Physics Data Facility, and the ACE", + "Principal Investigator, Edward C. Stone of the", + "California Institute of Technology, for usage of ACE", + "data from this site in publications and presentations.")) + +refs = {'mission': ' '.join(('Stone, E., Frandsen, A., Mewaldt, R. et al.', + 'The Advanced Composition Explorer. Space Science', + 'Reviews 86, 1–22 (1998).', + 'https://doi.org/10.1023/A:1005082526237')), + 'epam_l2': ' '.join(('Gold, R., Krimigis, S., Hawkins, S. et al.', + 'Electron, Proton, and Alpha Monitor on the', + 'Advanced Composition Explorer spacecraft.', + 'Space Science Reviews 86, 541–562 (1998).', + 'https://doi.org/10.1023/A:1005088115759')), + 'mag_l2': ' '.join(("Smith, C., L'Heureux, J., Ness, N. et al. The ACE", + "Magnetic Fields Experiment. Space Science Reviews", + "86, 613–632 (1998).", + "https://doi.org/10.1023/A:1005092216668")), + 'sis_l2': ' '.join(('Stone, E., Cohen, C., Cook, W. et al. The Solar', + 'Isotope Spectrometer for the Advanced Composition', + 'Explorer. Space Science Reviews 86, 357–408', + '(1998). https://doi.org/10.1023/A:1005027929871')), + 'swepam_l2': ' '.join(('McComas, D., Bame, S., Barker, P. et al. Solar', + 'Wind Electron Proton Alpha Monitor (SWEPAM)', + 'for the Advanced Composition Explorer. Space', + 'Science Reviews 86, 563–612 (1998).', + 'https://doi.org/10.1023/A:1005040232597')) + } + + +def load(fnames, tag='', inst_id='', to_pandas=False, **kwargs): + """Load ACE data via xarray and convert to pandas if needed. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + to_pandas : bool + If True, convert to pandas. If False, leave as xarray. (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + A pandas DataFrame or xarray Dataset with data prepared for the + `pysat.Instrument`. + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Note + ---- + Several variables relating to time stored in different formats are dropped. + These are redundant and complicate the load procedure. + + """ + + meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} + data, meta = cdw.load(fnames, tag=tag, inst_id=inst_id, pandas_format=False, + meta_translation=meta_translation, + drop_dims=['dim_empty', 'dim0', 'unit_time'], + use_cdflib=True) + + if to_pandas: + if hasattr(data, 'to_pandas'): + data = data.to_pandas() + else: + # xarray 0.16 support required for operational server + data = data.to_dataframe() + + return data, meta diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index c71115f2..235090eb 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -7,15 +7,16 @@ """ -import cdflib import datetime as dt +import numpy as np import os import pandas as pds import requests -import warnings +from time import sleep import xarray as xr from bs4 import BeautifulSoup +from cdasws import CdasWs import pysat from pysat.instruments.methods import general @@ -24,6 +25,13 @@ from pysat.utils import io from pysatNASA.instruments.methods import CDF as libCDF +try: + # cdflib 1.0 syntax + from cdflib.xarray import cdf_to_xarray +except ModuleNotFoundError: + # cdflib 0.4 syntax required for backwards compatibility + from cdflib import cdf_to_xarray + try: import pysatCDF auto_CDF = pysatCDF.CDF @@ -31,10 +39,39 @@ auto_CDF = libCDF +def try_inst_dict(inst_id, tag, supported_tags): + """Check that the inst_id and tag combination is valid. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + + Returns + ------- + inst_dict : dict or str + dictionary containing file location in spdf archive, or dataset ID for + cdasws + """ + try: + inst_dict = supported_tags[inst_id][tag] + except KeyError: + raise ValueError('inst_id / tag combo unknown.') + + return inst_dict + + def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=True, pandas_format=True, epoch_name='Epoch', - meta_processor=None, meta_translation=None, drop_meta_labels=None, - use_cdflib=None): + drop_dims=None, var_translation=None, meta_processor=None, + meta_translation=None, drop_meta_labels=None, use_cdflib=None): """Load NASA CDAWeb CDF files. Parameters @@ -62,6 +99,13 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), specified by `epoch_origin` with units specified by `epoch_unit`. This epoch variable will be converted to a `DatetimeIndex` for consistency across pysat instruments. (default='Epoch') + drop_dims : list or NoneType + List of variable dimensions that should be dropped. Applied + to data as loaded from the file. Used only from xarray Dataset. + (default=None) + var_translation : dict or NoneType + Variables that should be renamed. Applied to data as loaded + from the file. Used only from xarray Dataset. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -93,8 +137,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), Note ---- - This routine is intended to be used by pysat instrument modules supporting - a particular NASA CDAWeb dataset + - This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset + - pysatCDF (as of v0.3.2) does not support numpy>=1.24. Load errors may + arise. See https://github.com/pysat/pysatCDF/issues/46 """ @@ -106,10 +152,13 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), else: if not use_cdflib: estr = 'The `use_cdflib` option is not currently enabled for xarray' - warnings.warn(estr) + pysat.logger.warn(estr) data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, + drop_dims=drop_dims, + var_translation=var_translation, + file_cadence=file_cadence, meta_processor=meta_processor, meta_translation=meta_translation, drop_meta_labels=drop_meta_labels) @@ -224,15 +273,15 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), def load_xarray(fnames, tag='', inst_id='', - labels={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'plot': ('plot_label', str), 'axis': ('axis', str), - 'scale': ('scale', str), - 'min_val': ('value_min', float), - 'max_val': ('value_max', float), - 'fill_val': ('fill', float)}, - epoch_name='Epoch', meta_processor=None, - meta_translation=None, drop_meta_labels=None): + file_cadence=dt.timedelta(days=1), + labels={'units': ('Units', str), 'name': ('Long_Name', str), + 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), + 'min_val': ('ValidMin', (float, int, str)), + 'max_val': ('ValidMax', (float, int, str)), + 'fill_val': ('FillVal', (float, int, str))}, + epoch_name='Epoch', drop_dims=None, var_translation=None, + meta_processor=None, meta_translation=None, + drop_meta_labels=None): """Load NASA CDAWeb CDF files into an xarray Dataset. Parameters @@ -243,20 +292,31 @@ def load_xarray(fnames, tag='', inst_id='', Data product tag (default='') inst_id : str Instrument ID (default='') + file_cadence : dt.timedelta or pds.DateOffset + pysat assumes a daily file cadence, but some instrument data files + contain longer periods of time. This parameter allows the specification + of regular file cadences greater than or equal to a day (e.g., weekly, + monthly, or yearly). (default=dt.timedelta(days=1)) labels : dict Dict where keys are the label attribute names and the values are tuples that have the label values and value types in that order. - (default={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', np.float64), - 'max_val': ('value_max', np.float64), - 'fill_val': ('fill', np.float64)}) + (default={'units': ('Units', str), 'name': ('Long_Name', str), + 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), + 'min_val': ('ValidMin', (float, int, str)), + 'max_val': ('ValidMax', (float, int, str)), + 'fill_val': ('FillVal', (float, int, str))}) epoch_name : str Data key for epoch variable. The epoch variable is expected to be an array of integer or float values denoting time elapsed from an origin specified by `epoch_origin` with units specified by `epoch_unit`. This epoch variable will be converted to a `DatetimeIndex` for consistency across pysat instruments. (default='Epoch') + drop_dims : list or NoneType + List of variable dimensions that should be dropped. Applied + to data as loaded from the file. (default=None) + var_translation : dict or NoneType + Variables that should be renamed. Applied to data as loaded + from the file. Used only from xarray Dataset. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -308,33 +368,27 @@ def load_xarray(fnames, tag='', inst_id='', # metadata for pysat using some assumptions. Depending upon your needs # the resulting pandas DataFrame may need modification. ldata = [] - for lfname in fnames: - temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) + + # Find unique files for monthly / yearly cadence. + # Arbitrary timestamp needed for comparison. + t0 = dt.datetime(2009, 1, 1) + if (t0 + file_cadence) > (t0 + dt.timedelta(days=1)): + lfnames = list(np.unique([fname[:-11] for fname in fnames])) + else: + lfnames = fnames + + for lfname in lfnames: + temp_data = cdf_to_xarray(lfname, to_datetime=True) + if drop_dims: + temp_data = temp_data.drop_dims(drop_dims) + if var_translation: + temp_data = temp_data.rename(var_translation) ldata.append(temp_data) - # Combine individual files together + # Combine individual files together, concat along epoch if len(ldata) > 0: - data = xr.combine_by_coords(ldata) - - all_vars = io.xarray_all_vars(data) - - # Convert output epoch name to 'time' for pysat consistency - if epoch_name != 'time': - if 'time' not in all_vars: - if epoch_name in data.dims: - data = data.rename({epoch_name: 'time'}) - elif epoch_name in all_vars: - data = data.rename({epoch_name: 'time'}) - wstr = ''.join(['Epoch label: "', epoch_name, '"', - ' is not a dimension.']) - pysat.logger.warning(wstr) - else: - estr = ''.join(['Epoch label: "', epoch_name, '"', - ' not found in loaded data, ', - repr(all_vars)]) - raise ValueError(estr) - - epoch_name = 'time' + data = xr.combine_nested(ldata, epoch_name, + combine_attrs='override') all_vars = io.xarray_all_vars(data) @@ -390,15 +444,35 @@ def load_xarray(fnames, tag='', inst_id='', for key in filt_mdict: meta[key] = filt_mdict[key] + # Convert output epoch name to 'time' for pysat consistency + # This needs to be done last so that meta is updated properly + if epoch_name != 'time': + if 'time' not in all_vars: + if epoch_name in data.dims: + data = data.rename({epoch_name: 'time'}) + elif epoch_name in all_vars: + data = data.rename({epoch_name: 'time'}) + wstr = ''.join(['Epoch label: "', epoch_name, '"', + ' is not a dimension.']) + pysat.logger.warning(wstr) + else: + estr = ''.join(['Epoch label: "', epoch_name, '"', + ' not found in loaded data, ', + repr(all_vars)]) + raise ValueError(estr) + + epoch_name = 'time' + # Remove attributes from the data object data.attrs = {} return data, meta -def download(date_array, tag='', inst_id='', supported_tags=None, - remote_url='https://cdaweb.gsfc.nasa.gov', data_path=None): - """Download NASA CDAWeb CDF data. +# TODO(#103): Include support to unzip / untar files after download. +def download(date_array, data_path, tag='', inst_id='', supported_tags=None, + remote_url='https://cdaweb.gsfc.nasa.gov'): + """Download NASA CDAWeb data. This routine is intended to be used by pysat instrument modules supporting a particular NASA CDAWeb dataset. @@ -407,6 +481,8 @@ def download(date_array, tag='', inst_id='', supported_tags=None, ---------- date_array : array-like Array of datetimes to download data for. Provided by pysat. + data_path : str + Path to data directory. tag : str Data product tag (default='') inst_id : str @@ -419,15 +495,12 @@ def download(date_array, tag='', inst_id='', supported_tags=None, remote_url : str Remote site to download data from (default='https://cdaweb.gsfc.nasa.gov') - data_path : str or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) Examples -------- :: - # download support added to cnofs_vefi.py using code below + # Download support added to cnofs_vefi.py using code below fn = 'cnofs_vefi_bfield_1sec_{year:4d}{month:02d}{day:02d}_v05.cdf' dc_b_tag = {'remote_dir': ''.join(('/pub/data/cnofs/vefi/bfield_1sec', '/{year:4d}/')), @@ -439,10 +512,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, """ - try: - inst_dict = supported_tags[inst_id][tag] - except KeyError: - raise ValueError('inst_id / tag combo unknown.') + inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server remote_dir = inst_dict['remote_dir'] @@ -457,12 +527,21 @@ def download(date_array, tag='', inst_id='', supported_tags=None, # Download only requested files that exist remotely for date, fname in remote_files.items(): # Format files for specific dates and download location - formatted_remote_dir = remote_dir.format(year=date.year, - month=date.month, - day=date.day, - hour=date.hour, - min=date.minute, - sec=date.second) + # Year and day found in remote_dir: day is assumed to be day of year + if 'day' in remote_dir and 'month' not in remote_dir: + doy = date.timetuple().tm_yday + formatted_remote_dir = remote_dir.format(year=date.year, + day=doy, + hour=date.hour, + min=date.minute, + sec=date.second) + else: + formatted_remote_dir = remote_dir.format(year=date.year, + month=date.month, + day=date.day, + hour=date.hour, + min=date.minute, + sec=date.second) remote_path = '/'.join((remote_url.strip('/'), formatted_remote_dir.strip('/'), fname)) @@ -485,6 +564,104 @@ def download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File not available for', date.strftime('%d %B %Y')))) + # Pause to avoid excessive pings to server + sleep(0.2) + return + + +def cdas_download(date_array, data_path, tag='', inst_id='', + supported_tags=None): + """Download NASA CDAWeb CDF data using cdasws. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + data_path : str + Path to data directory. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be pre-set with + functools.partial then assigned to new instrument code. + (default=None) + + Note + ---- + Supported tags for this function use the cdaweb dataset naming convention. + You can find the data set names on CDAWeb or you can use cdasws. + + Starting from scratch using cdasws + :: + from cdasws import CdasWs + cdas = CdasWs() + + # Get list of available observatories/platforms. + cdas.get_observatories() + + # Once your observatory is located, list the available instruments. + cdas.get_instruments(observatory=‘observatory_name’) + + # Now list the available data sets for one instrument. + cdas.get_datasets(observatory=‘observatory_name’, + instrument=‘instrument_name’) + + # You can also list all of the datasets for an observatory. + cdas.get_datasets(observatory=‘observatory_name’) + + Alternatively + :: + Visit https://cdaweb.gsfc.nasa.gov/ + Select the observatory you want from the list and press submit. + The following page will have a list of the data sets. + The bolded names are in the format that cdasws uses. + + Examples + -------- + :: + # Download support added to cnofs_vefi.py using code below + download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} + download = functools.partial(cdw.cdas_download, + supported_tags=download_tags) + + """ + + start = date_array[0] + stop = date_array[-1] + remote_files = cdas_list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=supported_tags, + series_out=False) + + for file in remote_files: + + fname = file.split('/')[-1] + saved_local_fname = os.path.join(data_path, fname) + + # Perform download + logger.info(' '.join(('Attempting to download file: ', + file))) + try: + with requests.get(file) as req: + if req.status_code != 404: + with open(saved_local_fname, 'wb') as open_f: + open_f.write(req.content) + logger.info('Successfully downloaded {:}.'.format( + saved_local_fname)) + else: + logger.info(' '.join(('File: "', file, + '" is not available'))) + except requests.exceptions.RequestException as exception: + logger.info(' '.join((str(exception), '- File: "', file, + '" Is not available'))) + # Pause to avoid excessive pings to server + sleep(0.2) return @@ -551,10 +728,7 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, """ - try: - inst_dict = supported_tags[inst_id][tag] - except KeyError: - raise ValueError('inst_id / tag combo unknown.') + inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server format_str = '/'.join((inst_dict['remote_dir'].strip('/'), @@ -599,18 +773,27 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, stop = dt.datetime.now() if (stop is None) else stop if 'year' in search_dir['keys']: + url_list = [] if 'month' in search_dir['keys']: search_times = pds.date_range(start, stop + pds.DateOffset(months=1), freq='M') + for time in search_times: + subdir = format_dir.format(year=time.year, month=time.month) + url_list.append('/'.join((remote_url, subdir))) else: - search_times = pds.date_range(start, - stop + pds.DateOffset(years=1), - freq='Y') - url_list = [] - for time in search_times: - subdir = format_dir.format(year=time.year, month=time.month) - url_list.append('/'.join((remote_url, subdir))) + if 'day' in search_dir['keys']: + search_times = pds.date_range(start, stop + + pds.DateOffset(days=1), + freq='D') + else: + search_times = pds.date_range(start, stop + + pds.DateOffset(years=1), + freq='Y') + for time in search_times: + doy = int(time.strftime('%j')) + subdir = format_dir.format(year=time.year, day=doy) + url_list.append('/'.join((remote_url, subdir))) try: for top_url in url_list: for level in range(n_layers + 1): @@ -654,3 +837,82 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, stored_list = stored_list[mask] return stored_list + + +def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, + supported_tags=None, series_out=True): + """Return a list of every file for chosen remote data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + start : dt.datetime or NoneType + Starting time for file list. A None value will start with the first + file found. + (default=None) + stop : dt.datetime or NoneType + Ending time for the file list. A None value will stop with the last + file found. + (default=None) + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + series_out : bool + boolean to determine output type. True for pandas series of file names, + and False for a list of the full web address. + + Returns + ------- + file_list : list + A list containing the verified available files + + Note + ---- + Supported tags for this function use the cdaweb dataset naming convention. + You can find the dataset names on cdaweb or you can use cdasws. + + Examples + -------- + :: + download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} + list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) + + download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}} + list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) + """ + cdas = CdasWs() + dataset = try_inst_dict(inst_id, tag, supported_tags) + + if start is None and stop is None: + # Use the topmost directory without variables + start = cdas.get_inventory(identifier=dataset)[0].start + stop = cdas.get_inventory(identifier=dataset)[-1].end + elif stop is None: + stop = start + dt.timedelta(days=1) + elif start == stop: + stop = start + dt.timedelta(days=1) + + if isinstance(start, pds._libs.tslibs.timestamps.Timestamp): + start = start.tz_localize('utc') + stop = stop.tz_localize('utc') + + og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop) + + if series_out: + name_list = [os.path.basename(f['Name']) for f in og_files[1]] + t_stamp = [pds.Timestamp(f['StartTime'][:10]) for f in og_files[1]] + file_list = pds.Series(data=name_list, index=t_stamp) + else: + file_list = [f['Name'] for f in og_files[1]] + + return file_list diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index ea33e23b..8e44575d 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,7 +3,10 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', +refs = {'fpi': ' '.join(('Hays, P B, Killeen, T L, and Kennedy, B C.', + '"Fabry-Perot interferometer on Dynamics Explorer".', + 'Space Sci. Instrum., v. 5, p. 395-416, 1981.')), + 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', 'n. 4, p. 493, 1981.')), @@ -15,9 +18,17 @@ 'Lippincott, D. R. Zuccaro, B. J. Holt, L. H. Harmon,', 'and S. Sanatani, The retarding potential analyzer', 'for dynamics explorer-B, Space Sci. Instrum. 5,', - '503–510 (1981).')), + '503–510 (1981).\n', + 'Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R.', + 'Zuccaro, L. L. Harmon, B. J. Holt, J. E. Doherty, R.', + 'A. Power, The ion drift meter for Dynamics', + 'Explorer-B, Space Sci. Instrum., 5, 511, 1981.')), 'wats': ' '.join(('N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E.', 'Hedin, G. R. Carrignan, J. C. Maurer, The', 'Dynamics Explorer Wind and Temperature Spectrometer', - 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')) + 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')), + 'vefi': ' '.join(('Maynard, N. C., E. A. Bielecki, H. G. Burdick,', + 'Instrumentation for vector electric field', + 'measurements from DE-B, Space Sci. Instrum., 5,', + '523, 1981.')) } diff --git a/pysatNASA/instruments/methods/dmsp.py b/pysatNASA/instruments/methods/dmsp.py new file mode 100644 index 00000000..3f930c0b --- /dev/null +++ b/pysatNASA/instruments/methods/dmsp.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for the DMSP data.""" + +ackn_str = "".join(["This Defense Meteorological Satellite Program (DMSP) ", + "satellite data is provided through CDAWeb"]) + +refs = {'ssusi': ''.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', + ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', + 'David C. Humm, and Ching-I. Meng "Validation of ', + 'remote sensing products produced by the Special ', + 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', + 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', + '4485, Optical Spectroscopic Techniques, Remote ', + 'Sensing, and Instrumentation for Atmospheric and ', + 'Space Research IV, (30 January 2002); ', + 'doi:10.1117/12.454268'))} diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 60ead57f..cc6a24e9 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -1,6 +1,6 @@ """General methods for NASA instruments.""" -import warnings +import numpy as np import pysat @@ -23,15 +23,52 @@ def init(self, module, name): # Set acknowledgements self.acknowledgements = getattr(module, 'ackn_str') + + if hasattr(module, 'rules_url'): + self.acknowledgements.format(getattr(module, 'rules_url')[name]) + pysat.logger.info(self.acknowledgements) # Set references refs = getattr(module, 'refs') + try: + # See if there is a tag level reference + inst_refs = refs[name][self.tag] + except TypeError: + # No tag-level ref, use name-levele + inst_refs = refs[name] if 'mission' in refs.keys(): - self.references = '\n'.join((refs['mission'], refs[name])) + self.references = '\n'.join((refs['mission'], inst_refs)) + else: + self.references = inst_refs + + return + + +def clean(self): + """Clean data to the specified level. + + Note + ---- + Basic cleaning to replace fill values with NaN + + """ + + # Get a list of coords for the data + if self.pandas_format: + coords = [self.data.index.name] else: - self.references = refs[name] + coords = [key for key in self.data.coords.keys()] + + for key in self.variables: + # Skip over the coordinates when cleaning + if key not in coords: + fill = self.meta[key, self.meta.labels.fill_val] + # Replace fill with nan + fill_mask = self[key] == fill + self[key] = self.data[key].where(~fill_mask) + self.meta[key] = {self.meta.labels.fill_val: np.nan} return @@ -46,7 +83,7 @@ def clean_warn(self): 'none' No cleaning applied, routine not called in this case. """ - warnings.warn(' '.join(('No cleaning routines available for', - self.platform, self.name))) + pysat.logger.warning(' '.join(('No cleaning routines available for', + self.platform, self.name))) return diff --git a/pysatNASA/instruments/methods/gold.py b/pysatNASA/instruments/methods/gold.py deleted file mode 100644 index 9ac51312..00000000 --- a/pysatNASA/instruments/methods/gold.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for GOLD data.""" - -ack_str = ' '.join(('This is a data product from the NASA Global-scale', - 'Observations of the Limb and Disk (GOLD) mission, an', - 'Heliophysics Explorer mission of opportunity launched', - 'in January 2018.\n Responsibility of the mission', - 'science falls to the Principal Investigator, Dr.', - 'Richard Eastes at University of Colorado/LASP.\n', - 'Validation of the L1B data products falls to the', - 'instrument lead investigators/scientists.\n* EUV', - 'Dr. Bill McClintock\nValidation of the L2 data', - 'products falls to Computational Physics, Inc.\n* Dr.', - 'Jerry Lumpe\n (https://gold.cs.ucf.edu/).\nOverall', - 'validation of the products is overseen by the GOLD', - 'Project Scientist Dr. Alan Burns.\nUsers of these', - 'data should contact and acknowledge the Principal', - 'Investigator Dr. Richard Eastes and the party', - 'directly responsible for the data product and the', - 'NASA Explorers Project Office.')) -ref_str = ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et al.', - 'The Global-Scale Observations of the Limb and Disk', - '(GOLD) Mission. Space Sci Rev 212, 383–408 (2017).', - 'https://doi.org/10.1007/s11214-017-0392-2')) diff --git a/pysatNASA/instruments/methods/gps.py b/pysatNASA/instruments/methods/gps.py index bb11168e..684ab544 100644 --- a/pysatNASA/instruments/methods/gps.py +++ b/pysatNASA/instruments/methods/gps.py @@ -1,5 +1,11 @@ # -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for JPL ROTI data.""" +"""Provides non-instrument specific routines for JPL ROTI data. + +.. deprecated:: 0.0.5 + This module is now included in `methods.igs`. + This instrument will be removed in 0.1.0+ to reduce redundancy. + +""" ackn_str = ' '.join(("The GPS Total Electron Content (TEC) data", "produced by the International Global Navigation", diff --git a/pysatNASA/instruments/methods/igs.py b/pysatNASA/instruments/methods/igs.py new file mode 100644 index 00000000..63ad4a5d --- /dev/null +++ b/pysatNASA/instruments/methods/igs.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for JPL ROTI data.""" + +ackn_str = ' '.join(("The GPS Total Electron Content (TEC) data", + "produced by the International Global Navigation", + "Satellite Systems Service (IGS) Ionosphere Working", + "Group is provided through CDAWeb")) + +refs = {'mission': ' '.join(('Feltens J, Schaer S (1998) IGS Products for the', + 'Ionosphere, IGS Position Paper. In:', + 'Proceedings of the IGS analysis centers', + 'workshop, ESOC, Darmstadt, Germany,', + 'pp 225–232, 9–11 February')), + 'gps': {'tec': ' '.join(('M. Hernández-Pajares, J.M. Juan, J. Sanz, R.', + 'Orus, A. Garcia-Rigo, J. Feltens, A.', + 'Komjathy, S.C. Schaer, and A. Krankowski,', + 'The IGS VTEC maps: a reliable source of', + 'ionospheric information since 1998, Journal', + 'of Geodesy (2009) 83:263–275', + 'doi:10.1007/s00190-008-0266-1.\n', + 'Feltens, J., M. Angling, N. Jackson‐Booth,', + 'N. Jakowski, M. Hoque, M. Hernández‐Pajares,', + 'A. Aragón‐Àngel, R. Orús, and R. Zandbergen', + '(2011), Comparative testing of four', + 'ionospheric models driven with GPS', + 'measurements, Radio Sci., 46, RS0D12,', + 'doi:10.1029/2010RS004584.\n', + 'Peng Chen, Hang Liu, Yongchao Ma, Naiquan', + 'Zheng, Accuracy and consistency of different', + 'global ionospheric maps released by IGS', + 'ionosphere associate analysis centers,', + 'Advances in Space Research, Volume 65, Issue', + '1, 2020, Pages 163-174,', + 'doi:10.1016/j.asr.2019.09.042.\n')), + 'roti': ' '.join(('Pi, X., A. J. Mannucci, U. J.', + 'Lindqwister, and C. M. Ho, Monitoring of', + 'global ionospheric irregularities using', + 'the worldwide GPS network, Geophys. Res.', + 'Lett., 24, 2283, 1997.\n', + 'Pi, X., F. J. Meyer, K. Chotoo, Anthony', + 'Freeman, R. G. Caton, and C. T. Bridgwood,', + 'Impact of ionospheric scintillation on', + 'Spaceborne SAR observations studied using', + 'GNSS, Proc. ION-GNSS, pp.1998-2006,', + '2012.'))}} diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py new file mode 100644 index 00000000..1f92c802 --- /dev/null +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +"""Module for data sets created by JHU APL.""" + +import datetime as dt +import numpy as np +import pandas as pds +import xarray as xr + +from pysat.utils.coords import expand_xarray_dims +from pysat.utils.io import load_netcdf + + +def build_dtimes(data, var, epoch=None, epoch_var='time'): + """Build datetime objects from standard JHU APL time variables. + + Parameters + ---------- + data : xr.Dataset + Xarray dataset with time variables + var : str + Common string to identify desired year, day of year, and seconds of day + epoch : dt.datetime or NoneType + Epoch to subtract from data or NoneType to get seconds of day from + `data` (default=None) + epoch_var : str + Epoch variable containing time data that seconds of day will be + obtained from if `epoch` != None (default='time') + + Returns + ------- + dtimes : list-like + List of datetime objects + + """ + ykey = 'YEAR{:s}'.format(var) + dkey = 'DOY{:s}'.format(var) + skey = 'TIME{:s}'.format(var) + + if epoch is None: + hours = [int(np.floor(sec / 3600.0)) for sec in data[skey].values] + mins = [int(np.floor((sec - hours[i] * 3600) / 60.0)) + for i, sec in enumerate(data[skey].values)] + secs = [int(np.floor((sec - hours[i] * 3600 - mins[i] * 60))) + for i, sec in enumerate(data[skey].values)] + dtimes = [ + dt.datetime.strptime( + "{:4d}-{:03d}-{:02d}-{:02d}-{:02d}-{:06.0f}".format( + int(data[ykey].values[i]), int(data[dkey].values[i]), + hours[i], mins[i], secs[i], + (sec - hours[i] * 3600 - mins[i] * 60 - secs[i]) * 1.0e6), + '%Y-%j-%H-%M-%S-%f') + for i, sec in enumerate(data[skey].values)] + else: + dtimes = [ + dt.datetime.strptime("{:4d}-{:03d}".format( + int(data[ykey].values[i]), int(data[dkey].values[i])), '%Y-%j') + + (pds.to_datetime(etime).to_pydatetime() - epoch) + for i, etime in enumerate(data[epoch_var].values)] + + return dtimes + + +def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, + strict_dim_check=True): + """Load JHU APL EDR Aurora data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + Instrument ID name used to identify different instrument carriers + (default='') + pandas_format : bool + False for xarray format, True for pandas (default=False) + strict_dim_check : bool + Used for xarray data (`pandas_format` is False). If True, warn the user + that the desired epoch, 'TIME', is not present as a dimension in the + NetCDF file. If False, no warning is raised. (default=True) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Note + ---- + Logger warning 'Epoch label: TIME is not a dimension.' is raised due to + the data format and pysat file expectations. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', tag='edr-aur') + inst.load(2003, 1) + + """ + # Define the input variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + single_data = list() + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', + meta_kwargs={'labels': labels}, + pandas_format=pandas_format, + decode_times=False, + strict_dim_check=strict_dim_check) + + # Calculate the time for this data file. The pysat `load_netcdf` routine + # converts the 'TIME' parameter (seconds of day) into datetime using + # the UNIX epoch as the date offset + ftime = dt.datetime.strptime( + "{:4d}-{:03d}".format( + sdata['YEAR'].values.astype(int), + sdata['DOY'].values.astype(int)), '%Y-%j') + ( + pds.to_datetime(sdata['time'].values).to_pydatetime() + - dt.datetime(1970, 1, 1)) + + # Assign a datetime variable, making indexing possible + sdata['time'] = ftime + sdata = sdata.assign_coords( + {'time': sdata['time']}).expand_dims(dim='time') + + # Save the data in the file list + single_data.append(sdata) + + # Update the meta data + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing 'TIME', once possible + for var in mdata.keys(): + # Update the fill value, using information from the global header + mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + + # After loading all the data, determine which dimensions need to be + # expanded. Pad the data so that all dimensions are the same shape. + single_data = expand_xarray_dims(single_data, mdata, dims_equal=False) + + # Combine all the data, indexing along time + data = xr.combine_by_coords(single_data) + + return data, mdata + + +def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, + strict_dim_check=True, combine_times=False): + """Load JHU APL SDR data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + Instrument ID name used to identify different instrument carriers + (default='') + pandas_format : bool + False for xarray format, True for pandas (default=False) + strict_dim_check : bool + Used for xarray data (`pandas_format` is False). If True, warn the user + that the desired epoch, 'TIME_DAY', is not present as a dimension in the + NetCDF file. If False, no warning is raised. (default=True)``` + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Note + ---- + Logger warning 'Epoch label: TIME is not a dimension.' is raised due to + the data format and pysat file expectations. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', tag='edr-aur') + inst.load(2003, 1) + + """ + # Define the input variables and working variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + load_time = 'TIME_DAY' + time_vars = ['YEAR_DAY', 'DOY_DAY', 'TIME_EPOCH_DAY', 'YEAR_NIGHT', + 'DOY_NIGHT', 'TIME_NIGHT', 'TIME_EPOCH_NIGHT'] + coords = ['PIERCEPOINT_NIGHT_LATITUDE', 'PIERCEPOINT_NIGHT_LONGITUDE', + 'PIERCEPOINT_NIGHT_ALTITUDE', 'PIERCEPOINT_NIGHT_SZA', + 'PIERCEPOINT_DAY_LATITUDE', 'PIERCEPOINT_DAY_LONGITUDE', + 'PIERCEPOINT_DAY_ALTITUDE', 'PIERCEPOINT_DAY_SZA'] + time_dims = ['time'] + rename_dims = {'nAlongDay': 'nAlong', 'nAlongNight': 'nAlong'} + + if tag == 'sdr-imaging': + time_vars.extend(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", + "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) + coords.extend(['PIERCEPOINT_DAY_LATITUDE_AURORAL', + 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', + 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', + 'PIERCEPOINT_DAY_SZA_AURORAL']) + time_dims.append('time_auroral') + rename_dims['nCrossDay'] = 'nCross' + rename_dims['nCrossNight'] = 'nCross' + rename_dims['nAlongDayAur'] = 'time_auroral' + elif tag == 'sdr-spectrograph': + coords.extend(['PIERCEPOINT_NIGHT_ZENITH_ANGLE', + 'PIERCEPOINT_NIGHT_SAZIMUTH', + 'PIERCEPOINT_DAY_ZENITH_ANGLE', + 'PIERCEPOINT_DAY_SAZIMUTH']) + + if inst_id == 'low_res': + time_vars.extend(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", + "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", + "DOY_GAIM_NIGHT"]) + time_dims.extend(['time_gaim_day', 'time_gaim_night']) + rename_dims['nAlongGAIMDay'] = 'time_gaim_day' + rename_dims['nAlongGAIMNight'] = 'time_gaim_night' + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + inners = None + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', + meta_kwargs={'labels': labels}, + pandas_format=pandas_format, + decode_times=False, + strict_dim_check=strict_dim_check) + + # Calculate the time for this data file. The pysat `load_netcdf` routine + # converts the 'TIME' parameter (seconds of day) into datetime using + # the UNIX epoch as the date offset + ftime = build_dtimes(sdata, '_DAY', dt.datetime(1970, 1, 1)) + + # Ensure identical day and night dimensions + if sdata.dims['nAlongDay'] != sdata.dims['nAlongNight']: + raise ValueError('Along-track day and night dimensions differ') + + if 'nCrossDay' in rename_dims.keys(): + if sdata.dims['nCrossDay'] != sdata.dims['nCrossNight']: + raise ValueError('Cross-track day and night dimensions differ') + + # Combine identical dimensions and rename 'nAlong' to 'time' + sdata = sdata.rename_dims(rename_dims) + + if tag == 'sdr-imaging': + sdata = sdata.assign(time_auroral=build_dtimes(sdata, + '_DAY_AURORAL')) + elif tag == 'sdr-spectrograph' and inst_id == 'low_res': + sdata = sdata.assign(time_gaim_day=build_dtimes( + sdata, '_GAIM_DAY'), time_gaim_night=build_dtimes( + sdata, '_GAIM_NIGHT')) + + # Test that day and night times are consistent to the nearest second + for i, ntime in enumerate(build_dtimes(sdata, '_NIGHT')): + if abs(ntime - ftime[i]).total_seconds() > 1.0: + raise ValueError('Day and night times differ') + + # Remove redundant time variables and rname the 'nAlong' dimension + sdata = sdata.drop_vars(time_vars).swap_dims({'nAlong': 'time'}) + + # Assign time as a coordinate for combining files indexing + sdata['time'] = ftime + + # Separate into inner datasets + inner_keys = {dim: [key for key in sdata.keys() + if dim in sdata[key].dims] for dim in time_dims} + inner_dat = {dim: sdata.get(inner_keys[dim]) for dim in time_dims} + + # Add 'single_var's into 'time' dataset to keep track + sv_keys = [val.name for val in sdata.values() + if 'single_var' in val.dims] + singlevar_set = sdata.get(sv_keys) + inner_dat['time'] = xr.merge([inner_dat['time'], singlevar_set]) + + # Concatenate along desired dimension with previous files' data + if inners is None: + # No previous data, assign the data separated by dimension + inners = dict(inner_dat) + else: + # Concatenate with existing data + inners = {dim: xr.concat([inners[dim], inner_dat[dim]], dim=dim) + for dim in time_dims} + + # Update the meta data + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing dimensions and time, once possible + for var in mdata.keys(): + # Update the fill value, using information from the global header + mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + + # Combine all time dimensions + if combine_times: + data_list = expand_xarray_dims([inners[dim] if dim == 'time' else + inners[dim].rename_dims({dim: 'time'}) + for dim in time_dims], mdata, + dims_equal=False) + else: + data_list = [inners[dim] for dim in time_dims] + + # Combine all the data, indexing along time + data = xr.merge(data_list) + + # Set additional coordinates + data = data.set_coords(coords).assign_coords({'time': data['time']}) + if tag == 'sdr-imaging': + data = data.assign_coords( + {'nchan': ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], + "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", + "LBHlong"], + "nCross": sdata.nCross.data, + "nCrossDayAur": sdata.nCrossDayAur.data}) + elif tag == 'sdr-spectrograph': + data = data.assign_coords({"nchan": ["121.6nm", "130.4nm", "135.6nm", + "LBHshort", "LBHlong", "?"]}) + + # Ensure the data is ordered correctly + data = data.sortby('time') + + return data, mdata diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index 8640340a..2844b0cb 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -5,9 +5,8 @@ import numpy as np import pandas as pds from scipy import stats -import warnings -from pysat import logger +import pysat def time_shift_to_magnetic_poles(inst): @@ -46,8 +45,8 @@ def time_shift_to_magnetic_poles(inst): time_x = inst['BSN_x'] * 6371.2 / -inst['Vx'] idx, = np.where(np.isnan(time_x)) if len(idx) > 0: - logger.info(time_x[idx]) - logger.info(time_x) + pysat.logger.info(time_x[idx]) + pysat.logger.info(time_x) time_x_offset = [pds.DateOffset(seconds=time) for time in time_x.astype(int)] new_index = [] @@ -115,8 +114,10 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, max_wnum = np.floor(steady_window / sample_rate) if max_wnum != steady_window / sample_rate: steady_window = max_wnum * sample_rate - logger.warning("sample rate is not a factor of the statistical window") - logger.warning("new statistical window is {:.1f}".format(steady_window)) + pysat.logger.warning(" ".join(("sample rate is not a factor of the", + "statistical window"))) + pysat.logger.warning(" ".join(("new statistical window is", + "{:.1f}".format(steady_window)))) min_wnum = int(np.ceil(max_wnum * min_window_frac)) @@ -137,9 +138,9 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, kwargs=circ_kwargs, raw=True) except TypeError: - warnings.warn(' '.join(['To automatically remove NaNs from the', - 'calculation, please upgrade to scipy 1.4 or', - 'newer'])) + pysat.logger.warn(' '.join(['To automatically remove NaNs from the', + 'calculation, please upgrade to scipy 1.4', + 'or newer.'])) circ_kwargs.pop('nan_policy') ca_std = \ inst['clock_angle'].rolling(min_periods=min_wnum, diff --git a/pysatNASA/instruments/methods/ses14.py b/pysatNASA/instruments/methods/ses14.py new file mode 100644 index 00000000..5307d592 --- /dev/null +++ b/pysatNASA/instruments/methods/ses14.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for SES14 instruments.""" + +ackn_str = ' '.join(('This is a data product from the NASA Global-scale', + 'Observations of the Limb and Disk (GOLD) mission, an', + 'Heliophysics Explorer mission of opportunity launched', + 'in January 2018.\n Responsibility of the mission', + 'science falls to the Principal Investigator, Dr.', + 'Richard Eastes at University of Colorado/LASP.\n', + 'Validation of the L1B data products falls to the', + 'instrument lead investigators/scientists.\n* EUV', + 'Dr. Bill McClintock\nValidation of the L2 data', + 'products falls to Computational Physics, Inc.\n* Dr.', + 'Jerry Lumpe\n (https://gold.cs.ucf.edu/).\nOverall', + 'validation of the products is overseen by the GOLD', + 'Project Scientist Dr. Alan Burns.\nUsers of these', + 'data should contact and acknowledge the Principal', + 'Investigator Dr. Richard Eastes and the party', + 'directly responsible for the data product and the', + 'NASA Explorers Project Office.')) +refs = {'gold': ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et', + 'al., The Global-Scale Observations of the Limb', + 'and Disk (GOLD) Mission. Space Sci Rev 212,', + '383–408 (2017). doi:10.1007/s11214-017-0392-2'))} diff --git a/pysatNASA/instruments/methods/timed.py b/pysatNASA/instruments/methods/timed.py new file mode 100644 index 00000000..e6d4e2fa --- /dev/null +++ b/pysatNASA/instruments/methods/timed.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for the TIMED data.""" + +rules_url = {'guvi': 'http://guvitimed.jhuapl.edu/home_guvi-datausage', + 'saber': 'https://saber.gats-inc.com/data_services.php', + 'see': 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl'} + +ackn_str = "".join(["This Thermosphere Ionosphere Mesosphere Energetics ", + "Dynamics (TIMED) satellite data is provided through ", + "CDAWeb. Please see the Rules of the Road at {:s}"]) + +refs = {'guvi': ''.join(['Larry J. Paxton, Andrew B. Christensen, David C. ', + 'Humm, Bernard S. Ogorzalek, C. Thompson Pardoe, ', + 'Daniel Morrison, Michele B. Weiss, W. Crain, ', + 'Patricia H. Lew, Dan J. Mabry, John O. Goldsten, ', + 'Stephen A. Gary, David F. Persons, Mark J. Harold, ', + 'E. Brian Alvarez, Carl J. Ercol, Douglas J. ', + 'Strickland, and Ching-I. Meng "Global ultraviolet ', + 'imager (GUVI): measuring composition and energy ', + 'inputs for the NASA Thermosphere Ionosphere ', + 'Mesosphere Energetics and Dynamics (TIMED) mission",', + 'Proc. SPIE 3756, Optical Spectroscopic Techniques ', + 'and Instrumentation for Atmospheric and Space ', + 'Research III, (20 October 1999); ', + 'doi:10.1117/12.366380']), + 'saber': '', + 'see': ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', + 'S. M., Chamberlin, P. C., Lean, J.,', + 'Rottman, G. J., Solomon, S. C., Tobiska,', + 'W. K., and Woodraska, D. L. (2005),', + 'Solar EUV Experiment (SEE): Mission', + 'overview and first results, J. Geophys.', + 'Res., 110, A01312, doi:10.1029/2004JA010765.'))} diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index d709c1b5..d7a205cf 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -44,9 +44,8 @@ import pandas as pds import warnings +import pysat from pysat.instruments.methods import general as mm_gen -from pysat import logger -from pysat.utils import time as pysat_time from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import omni as mm_omni @@ -86,7 +85,7 @@ def init(self): 'magnetic field data, J. Geophys. Res.,', 'Vol. 110, No. A2, A02209,', '10.1029/2004JA010649.')) - logger.info(ackn_str) + pysat.logger.info(ackn_str) return @@ -140,59 +139,12 @@ def clean(self): file_cadence=pds.DateOffset(months=1)) # Set the list_remote_files routine -remote_dir = '/pub/data/omni/omni_cdaweb/hro_{tag:s}/{{year:4d}}/' -download_tags = {inst_id: {tag: {'remote_dir': remote_dir.format(tag=tag), - 'fname': supported_tags[inst_id][tag]} - for tag in tags.keys()} - for inst_id in inst_ids.keys()} -list_remote_files = functools.partial(cdw.list_remote_files, - supported_tags=download_tags) - - -# Set the download routine -def download(date_array, tag, inst_id, data_path, update_files=False): - """Download OMNI HRO data from CDAWeb. - - Parameters - ---------- - date_array : array-like - Sequence of dates for which files will be downloaded. - tag : str - Denotes type of file to load. - inst_id : str - Specifies the satellite ID for a constellation. - data_path : str - Path to data directory. - update_files : bool - Re-download data for files that already exist if True (default=False) - - Raises - ------ - IOError - If a problem is encountered connecting to the gateway or retrieving - data from the repository. - - Warnings - -------- - Only able to download current forecast data, not archived forecasts. - - Note - ---- - Called by pysat. Not intended for direct use by user. - - """ - - # Set the download tags - - # Adjust the date_array for monthly downloads - if date_array.freq != 'MS': - date_array = pysat_time.create_date_range( - dt.datetime(date_array[0].year, date_array[0].month, 1), - date_array[-1], freq='MS') +download_tags = {'': {'1min': 'OMNI_HRO_1MIN', '5min': 'OMNI_HRO_5MIN'}} +download = functools.partial(cdw.cdas_download, + supported_tags=download_tags) - cdw.download(date_array, tag=tag, inst_id=inst_id, - supported_tags=download_tags, data_path=data_path) - return +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) # Set the load routine diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 5a89413f..d9b1e9fb 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -11,12 +11,22 @@ 'gold' tag 'nmax' + 'tlimb' + 'tdisk' + 'o2den' +inst_id + None Supported Warnings -------- - The cleaning parameters for the instrument are still under development. -- strict_time_flag must be set to False +- Loading multiple days of data requires a bugfix in pysat 3.1.0 or higher. +Note +---- +In roughly 0.3% of daily files, Channel A and Channel B scans begin at the same +time. One microsecond is added to Channel B to ensure uniqueness in the xarray +index. The nominal scan rate for each channel is every 30 minutes. Examples -------- @@ -24,8 +34,7 @@ import datetime as dt import pysat - nmax = pysat.Instrument(platform='ses14', name='gold', tag='nmax' - strict_time_flag=False) + nmax = pysat.Instrument(platform='ses14', name='gold', tag='nmax') nmax.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) nmax.load(2020, 1) @@ -36,60 +45,38 @@ import numpy as np from pysat.instruments.methods import general as ps_gen -from pysat import logger from pysat.utils.io import load_netcdf from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa -from pysatNASA.instruments.methods import gold as mm_gold +from pysatNASA.instruments.methods import ses14 as mm_gold # ---------------------------------------------------------------------------- # Instrument attributes platform = 'ses14' name = 'gold' -tags = {'nmax': 'Level 2 Nmax data for the GOLD instrument'} -inst_ids = {'': ['nmax']} +tags = {'nmax': 'Level 2 max dens data for the GOLD instrument', + 'tlimb': 'Level 2 limb temp data for the GOLD instrument', + 'tdisk': 'Level 2 disk temp data for the GOLD instrument', + 'o2den': 'Level 2 O2 dens data for the GOLD instrument'} +inst_ids = {'': [tag for tag in tags.keys()]} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(2020, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - Parameters - ----------- - self : pysat.Instrument - Instrument class object - - """ - - logger.info(mm_gold.ack_str) - logger.warning(' '.join(('Time stamps may be non-unique because Channel A', - 'and B are different instruments. An upgrade to', - 'the pysat.Constellation object is required to', - 'solve this issue. See pysat issue #614 for more', - 'info.'))) - self.acknowledgements = mm_gold.ack_str - self.references = mm_gold.ref_str - - return - +init = functools.partial(mm_nasa.init, module=mm_gold, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # @@ -103,17 +90,90 @@ def init(self): list_files = functools.partial(ps_gen.list_files, supported_tags=supported_tags) +# Set download tags. Note that tlimb uses the general implementation, while +# other tags use the cdasws implementation. +download_tags = {'': {'tlimb': {'remote_dir': ''.join(('/pub/data/gold/', + 'level2/tlimb', + '/{year:4d}/')), + 'fname': supported_tags['']['tlimb']}, + 'nmax': 'GOLD_L2_NMAX', + 'o2den': 'GOLD_L2_O2DEN', + 'tdisk': 'GOLD_L2_TDISK'}} + + # Set the download routine -download_tags = {inst_id: - {tag: {'remote_dir': ''.join(('/pub/data/gold/level2/', tag, - '/{year:4d}/')), - 'fname': supported_tags[''][tag]} - for tag in tags.keys()} for inst_id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +def download(date_array, tag='', inst_id='', data_path=None): + """Download NASA GOLD data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + data_path : str or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + + """ + + if tag == 'tlimb': + cdw.download(date_array, tag=tag, inst_id=inst_id, + supported_tags=download_tags, data_path=data_path) + else: + cdw.cdas_download(date_array, tag=tag, inst_id=inst_id, + supported_tags=download_tags, data_path=data_path) + # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, - supported_tags=download_tags) +def list_remote_files(tag='', inst_id='', start=None, stop=None, + series_out=True): + """List every file for remote GOLD data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + start : dt.datetime or NoneType + Starting time for file list. A None value will start with the first + file found. + (default=None) + stop : dt.datetime or NoneType + Ending time for the file list. A None value will stop with the last + file found. + (default=None) + series_out : bool + boolean to determine output type. True for pandas series of file names, + and False for a list of the full web address. + (default=True) + + Returns + ------- + file_list : list + A list containing the verified available files + + """ + + if tag == 'tlimb': + file_list = cdw.list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=download_tags) + else: + file_list = cdw.cdas_list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=download_tags, + series_out=series_out) + return file_list def load(fnames, tag='', inst_id=''): @@ -179,15 +239,32 @@ def load(fnames, tag='', inst_id=''): 'Valid_Max': 'Valid_Max', '_FillValue': 'fill', 'FillVal': 'fill', 'TIME_BASE': 'time_base'} + if tag in ['nmax', 'tdisk', 'tlimb']: + epoch_name = 'nscans' + + elif tag == 'o2den': + epoch_name = 'nevents' + data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name='nscans', labels=labels, + epoch_name=epoch_name, + meta_kwargs={'labels': labels}, meta_translation=meta_translation, - drop_meta_labels='FILLVAL') + combine_by_coords=False, + drop_meta_labels='FILLVAL', + decode_times=False) - if tag == 'nmax': + if tag in ['nmax', 'tdisk', 'tlimb']: # Add time coordinate from scan_start_time - data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") - for val in data['scan_start_time'].values] + time = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") + for val in data['scan_start_time'].values] + + # Add a delta of 1 microsecond for channel B. + delta_time = [1 if ch == b'CHB' else 0 for ch in data['channel'].values] + data['time'] = [time[i] + dt.timedelta(microseconds=delta_time[i]) + for i in range(0, len(time))] + + # Sort times to ensure monotonic increase. + data = data.sortby('time') # Update coordinates with dimensional data data = data.assign_coords({'nlats': data['nlats'], @@ -200,4 +277,30 @@ def load(fnames, tag='', inst_id=''): meta['nlons'] = {meta.labels.notes: 'Index for longitude values'} meta['nmask'] = {meta.labels.notes: 'Index for mask values'} + elif tag == 'o2den': + + # Removing extra variables + if len(data['zret'].dims) > 1: + data['zret'] = data['zret'].isel(time=0) + data['zdat'] = data['zdat'].isel(time=0) + + # Add time coordinate from utc_time + data['time'] = [dt.datetime.strptime(str(val), + "b'%Y-%m-%dT%H:%M:%S.%fZ'") + for val in data['time_utc'].values] + + # Add retrieval altitude values and data tangent altitude values + data = data.swap_dims({"nzret": "zret", "nzdat": "zdat"}) + + # Update coordinates with dimensional data + data = data.assign_coords({'zret': data['zret'], + 'zdat': data['zdat'], + 'n_wavelength': data['n_wavelength'], + 'channel': data['channel']}) + meta['time'] = {meta.labels.notes: 'Converted from time_utc'} + meta['zret'] = {meta.labels.notes: ''.join(('Index for retrieval', + ' altitude values'))} + meta['zdat'] = {meta.labels.notes: ''.join(('Index for data tangent', + ' altitude values'))} + return data, meta diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py new file mode 100644 index 00000000..2a0f7da8 --- /dev/null +++ b/pysatNASA/instruments/timed_guvi.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +"""Module for the TIMED GUVI instrument. + +Supports the Global UltraViolet Imager (GUVI) instrument on the Thermosphere +Ionosphere Mesosphere Energetics Dynamics (TIMED) satellite data from the +NASA Coordinated Data Analysis Web (CDAWeb). + +From JHU APL: + +The Global Ultraviolet Imager (GUVI) is one of four instruments that constitute +the TIMED spacecraft, the first mission of the NASA Solar Connections program. +The TIMED spacecraft is being built by Johns Hopkins University Applied Physics +Laboratory and GUVI is a joint collaboration between JHU/APL and the Aerospace +Corporation. TIMED will be used to study the energetics and dynamics of the +Mesosphere and lower Thermosphere between an altitude of approximately 60 to 180 +kilometers. + +References +---------- +Larry J. Paxton, Andrew B. Christensen, David C. Humm, Bernard S. Ogorzalek, C. +Thompson Pardoe, Daniel Morrison, Michele B. Weiss, W. Crain, Patricia H. Lew, +Dan J. Mabry, John O. Goldsten, Stephen A. Gary, David F. Persons, Mark J. +Harold, E. Brian Alvarez, Carl J. Ercol, Douglas J. Strickland, and Ching-I. +Meng "Global ultraviolet imager (GUVI): measuring composition and energy inputs +for the NASA Thermosphere Ionosphere Mesosphere Energetics and Dynamics (TIMED) +mission", Proc. SPIE 3756, Optical Spectroscopic Techniques and Instrumentation +for Atmospheric and Space Research III, (20 October 1999); +https://doi.org/10.1117/12.366380 + +Properties +---------- +platform + 'timed' +name + 'guvi' +tag + 'edr-aur' + 'sdr-imaging' + 'sdr-spectrograph' +inst_id + '' + 'high_res' + 'low_res' + +Warnings +-------- +- Currently no cleaning routine. + +Example +------- +:: + + import pysat + guvi = pysat.Instrument(platform='timed', name='guvi', + inst_id='sdr-imaging', tag='low_res') + guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) + guvi.load(date=dt.datetime(2005, 6, 28)) + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import jhuapl +from pysatNASA.instruments.methods import timed as mm_timed + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'timed' +name = 'guvi' +tags = {'edr-aur': 'Level 2 Auroral disk imaging mode', + 'sdr-imaging': 'Level 1C imaging data', + 'sdr-spectrograph': 'Level 1C spectrograph data'} +inst_ids = {'': ['edr-aur'], + 'high_res': ['sdr-imaging', 'sdr-spectrograph'], + 'low_res': ['sdr-imaging', 'sdr-spectrograph']} + +pandas_format = False +multi_file_day = True + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]} + for iid in inst_ids.keys()} +_test_load_opt = {iid: {tag: {'combine_times': True} + for tag in inst_ids[iid]} for iid in ['high_res', + 'low_res']} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('TIMED_GUVI_{lvl:s}{mode:s}_{{year:04d}}{{day:03d}}', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-?????????????_REV', + '??????_Av{{version:02d}}-??r{{revision:03d}}.nc')) +file_lvl = {'low_res': 'L1C-2-disk', 'high_res': 'L1C-disk', '': 'L2B'} +mode = {'sdr-imaging': '-IMG', 'sdr-spectrograph': '-SPECT', + 'edr-aur': '-edr-aur-IMG'} +supported_tags = {inst_id: {tag: fname.format(lvl=file_lvl[inst_id], + mode=mode[tag]) + for tag in tags.keys()} + for inst_id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) + +# Set the download routine +url = ''.join(('/pub/data/timed/guvi/levels_v13/{lvl:s}/{mode:s}/', + '{{year:4d}}/{{day:03d}}/')) +url_lvl = {'sdr-imaging': 'level1c', 'sdr-spectrograph': 'level1c', + 'edr-aur': 'level2b'} +url_mode = {tag: 'imaging/edr-aur' if tag == 'edr-aur' else tag.split('-')[1] + for tag in tags.keys()} +download_tags = {iid: {tag: {'remote_dir': url.format(lvl=url_lvl[tag], + mode=url_mode[tag]), + 'fname': fname.format(lvl=file_lvl[iid], + mode=mode[tag])} + for tag in tags.keys()} for iid in inst_ids.keys()} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) + + +# Set the load routine +def load(fnames, tag='', inst_id='', combine_times=False): + """Load TIMED GUVI data into `xarray.DataSet` and `pysat.Meta` objects. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + inst_id : str + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) + + Returns + ------- + data : xr.DataSet + A xarray DataSet with data prepared for the pysat.Instrument + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Raises + ------ + ValueError + If temporal dimensions are not consistent + + Note + ---- + Any additional keyword arguments passed to pysat.Instrument + upon instantiation are passed along to this routine. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', + inst_id='high_res', tag='sdr-imaging') + inst.load(2005, 179) + + """ + if tag == 'edr-aur': + data, meta = jhuapl.load_edr_aurora(fnames, tag, inst_id, + pandas_format=pandas_format, + strict_dim_check=False) + else: + data, meta = jhuapl.load_sdr_aurora(fnames, tag, inst_id, + pandas_format=pandas_format, + strict_dim_check=False, + combine_times=combine_times) + + return data, meta diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 38f82b91..e1e5086a 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -50,10 +50,10 @@ # CDAWeb methods prewritten for pysat from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- # Instrument attributes @@ -68,7 +68,7 @@ # Set to False to specify using xarray (not using pandas) # Set to True if data will be returned via a pandas DataFrame -pandas_format = True +pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -78,33 +78,18 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - rules_url = 'https://saber.gats-inc.com/data_services.php' - ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = '' - - return - +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # # Use the default CDAWeb and pysat methods +# TODO(#104): Switch to netCDF4 files once unzip (#103) is supported. + # Set the list_files routine fname = ''.join(('timed_l2a_saber_{year:04d}{month:02d}{day:02d}', '{hour:02d}{minute:02d}_v{version:02d}-{revision:02d}-', @@ -113,16 +98,17 @@ def init(self): list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) -# Set the load routine -load = cdw.load +# Set the load routine. Note that the time variable associated with +# tpaltitude is renamed to avoid conflict with renaming Epoch. +load = functools.partial(cdw.load, pandas_format=pandas_format, + drop_dims='record0', + var_translation={'time': 'tp_time'}, + use_cdflib=True) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/timed/saber/level2a_cdf', - '/{year:4d}/{month:02d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'TIMED_L2A_SABER'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 903058ab..02cc5d3c 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -17,19 +17,12 @@ None inst_id None supported -flatten_twod - If True, then two dimensional data is flattened across - columns. Name mangling is used to group data, first column - is 'name', last column is 'name_end'. In between numbers are - appended 'name_1', 'name_2', etc. All data for a given 2D array - may be accessed via, data.loc[:, 'item':'item_end'] - If False, then 2D data is stored as a series of DataFrames, - indexed by Epoch. data.loc[0, 'item'] - (default=True) Note ---- - no tag required +- cdflib load routine raises ISTP Compliance Warnings for several variables. + This is due to how the Epoch is listed in the original files. Warnings -------- @@ -42,10 +35,10 @@ import pandas as pds from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- # Instrument attributes @@ -54,6 +47,7 @@ name = 'see' tags = {'': ''} inst_ids = {'': [tag for tag in tags.keys()]} +pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -63,39 +57,18 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - rules_url = 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl' - ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', - 'S. M., Chamberlin, P. C., Lean, J.,', - 'Rottman, G. J., Solomon, S. C., Tobiska,', - 'W. K., and Woodraska, D. L. (2005),', - 'Solar EUV Experiment (SEE): Mission', - 'overview and first results, J. Geophys.', - 'Res., 110, A01312,', - 'doi:10.1029/2004JA010765.')) - - return - +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # # Use the default CDAWeb and pysat methods +# TODO(#104): Switch to netCDF4 files once unzip (#103) is supported. + # Set the list_files routine fname = 'timed_l3a_see_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' supported_tags = {'': {'': fname}} @@ -104,15 +77,13 @@ def init(self): file_cadence=pds.DateOffset(months=1)) # Set the load routine -load = functools.partial(cdw.load, file_cadence=pds.DateOffset(months=1)) +load = functools.partial(cdw.load, file_cadence=pds.DateOffset(months=1), + pandas_format=pandas_format, use_cdflib=True) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/timed/see/data/level3a_cdf', - '/{year:4d}/{month:02d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'TIMED_L3A_SEE'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 07e850f4..8b8ffb13 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -10,11 +10,23 @@ import pytest +# Import the test classes from pysat +import pysat +from pysat.tests.classes import cls_instrument_library as clslib +from pysat.utils import testing + # Make sure to import your instrument library here import pysatNASA -# Import the test classes from pysat -from pysat.tests.classes import cls_instrument_library as clslib + +try: + import pysatCDF # noqa: F401 + # If this successfully imports, tests need to be run with both pysatCDF + # and cdflib + cdflib_only = False +except ImportError: + # pysatCDF is not present, standard tests default to cdflib. + cdflib_only = True # Tell the standard tests which instruments to run each test on. @@ -27,7 +39,9 @@ for inst in instruments['download']: fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']] if '.cdf' in fname: - instruments['cdf'].append(inst) + temp_inst, _ = clslib.initialize_test_inst_and_date(inst) + if temp_inst.pandas_format: + instruments['cdf'].append(inst) class TestInstruments(clslib.InstLibTests): @@ -42,6 +56,9 @@ class TestInstruments(clslib.InstLibTests): @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) + @pytest.mark.skipif(cdflib_only, + reason=" ".join(("Additional load tests not required", + "when pysatCDF not installed"))) def test_load_cdflib(self, inst_dict): """Test that instruments load at each cleaning level. @@ -71,7 +88,7 @@ def test_load_cdflib(self, inst_dict): assert UserWarning in categories else: # If error message does not match, raise error anyway - raise(verr) + raise ValueError(verr) # Make sure fake data is cleared assert target not in test_inst.data @@ -79,3 +96,79 @@ def test_load_cdflib(self, inst_dict): pytest.skip("Download data not available.") return + + # TODO(https://github.com/pysat/pysat/issues/1020): This test should be + # removed when header level data is tested in version 3.2.0+ of pysat + @pytest.mark.second + @pytest.mark.parametrize("inst_dict", instruments['cdf']) + def test_meta_header(self, inst_dict): + """Test that instruments have header level metadata attached. + + Parameters + ---------- + inst_dict : dict + Dictionary containing info to instantiate a specific instrument. + + """ + test_inst, date = clslib.initialize_test_inst_and_date(inst_dict) + try: + test_inst.load(date=date, use_header=True, use_cdflib=True) + except ValueError as verr: + # Check if instrument is failing due to strict time flag + if str(verr).find('Loaded data') > 0: + test_inst.strict_time_flag = False + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True, use_cdflib=True) + + assert len(war) >= 1 + categories = [war[j].category for j in range(0, len(war))] + assert UserWarning in categories + else: + # If error message does not match, raise error anyway + raise (verr) + assert test_inst.meta.to_dict() != {} + + +class TestDeprecation(object): + """Unit test for deprecation warnings.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + + warnings.simplefilter("always", DeprecationWarning) + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + return + + @pytest.mark.parametrize("inst_module,tag", [('jpl_gps', 'roti')]) + def test_deprecated_instruments(self, inst_module, tag): + """Check that instantiating old instruments raises a DeprecationWarning. + + Parameters + ---------- + inst_module : str + name of deprecated module. + tag : str + tag of depracted instrument. + + """ + + with warnings.catch_warnings(record=True) as war: + pysat.Instrument(inst_module=getattr(pysatNASA.instruments, + inst_module), + tag=tag, use_header=True) + + warn_msgs = [" ".join(["The instrument module", + "`{:}`".format(inst_module), + "has been deprecated and will be removed", + "in 0.1.0+."])] + + # Ensure the minimum number of warnings were raised. + assert len(war) >= len(warn_msgs) + + # Test the warning messages, ensuring each attribute is present. + testing.eval_warnings(war, warn_msgs) + return diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index deb25417..dd7870e6 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -1,6 +1,7 @@ """Unit tests for the cdaweb instrument methods.""" import datetime as dt +import pandas as pds import requests import pytest @@ -16,8 +17,9 @@ class TestCDAWeb(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.download_tags = pysatNASA.instruments.cnofs_plp.download_tags + self.download_tags = pysatNASA.instruments.timed_guvi.download_tags self.kwargs = {'tag': None, 'inst_id': None} + self.saved_path = pysat.params['data_dirs'] return def teardown_method(self): @@ -31,7 +33,7 @@ def test_remote_file_list_connection_error_append(self): with pytest.raises(Exception) as excinfo: # Giving a bad remote_site address yields similar ConnectionError - cdw.list_remote_files(tag='', inst_id='', + cdw.list_remote_files(tag='sdr-imaging', inst_id='high_res', supported_tags=self.download_tags, remote_url='https://bad/path') @@ -60,6 +62,7 @@ def test_bad_kwarg_download(self, bad_key, bad_val, err_msg): with pytest.raises(ValueError) as excinfo: cdw.download(supported_tags=self.download_tags, date_array=date_array, + data_path=self.saved_path, tag=self.kwargs['tag'], inst_id=self.kwargs['inst_id']) assert str(excinfo.value).find(err_msg) >= 0 @@ -80,11 +83,33 @@ def test_bad_kwarg_list_remote_files(self, bad_key, bad_val, err_msg): assert str(excinfo.value).find(err_msg) >= 0 return - def test_remote_file_list_all(self): - """Test that remote_file_list works if start/stop dates unspecified.""" + @pytest.mark.parametrize("start, stop", + [(None, None), + (dt.datetime(2009, 1, 1), None), + (dt.datetime(2009, 1, 1), + dt.datetime(2009, 1, 1)), + (pds.Timestamp(2009, 1, 1), + pds.Timestamp(2009, 1, 2))]) + def test_remote_file_list_all(self, start, stop): + """Test that remote_file_list works for all start and stop cases.""" self.module = pysatNASA.instruments.cnofs_plp self.test_inst = pysat.Instrument(inst_module=self.module) - files = self.test_inst.remote_file_list() + files = self.test_inst.remote_file_list(start, stop) assert len(files) > 0 return + + @pytest.mark.parametrize("series_out", [(True), (False)]) + def test_cdas_remote_files(self, series_out): + """Test that cdas_list_remote_files can return pandas series.""" + start = dt.datetime(2009, 1, 1) + stop = dt.datetime(2009, 1, 2) + self.module = pysatNASA.instruments.cnofs_plp + self.test_inst = pysat.Instrument(inst_module=self.module) + files = self.test_inst.remote_file_list(start, stop, + series_out=series_out) + if series_out is True: + assert isinstance(files, pds.Series) + else: + assert isinstance(files, list) + return diff --git a/pysatNASA/tests/test_methods_platform.py b/pysatNASA/tests/test_methods_platform.py new file mode 100644 index 00000000..a1cd6611 --- /dev/null +++ b/pysatNASA/tests/test_methods_platform.py @@ -0,0 +1,129 @@ +"""Unit tests for the common NASA platform method attributes.""" + + +from pysatNASA.instruments import methods + + +class TestTIMEDMethods(object): + """Unit tests for `pysat.instruments.methods.timed`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['see', 'saber', 'guvi'] + self.module = methods.timed + self.platform_str = '(TIMED)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + def test_ack(self): + """Test that the acknowledgements reference the correct platform.""" + + assert self.module.ackn_str.find(self.platform_str) >= 0 + return + + def test_rules(self): + """Test that the rules of the road exist for each instrument.""" + + if hasattr(self.module, "rules_url"): + for name in self.names: + assert name in self.module.rules_url.keys( + ), "No rules URL for {:}".format(name) + return + + def test_ref(self): + """Test that all instruments have references.""" + + for name in self.names: + assert name in self.module.refs.keys( + ), "No reference for {:}".format(name) + return + + +class TestDMSPMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.dmsp`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ssusi'] + self.module = methods.dmsp + self.platform_str = '(DMSP)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestCNOFSMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.cnofs`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ivm', 'plp', 'vefi'] + self.module = methods.cnofs + self.platform_str = '(C/NOFS)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestDE2Methods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.de2`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['lang', 'nacs', 'rpa', 'wats'] + self.module = methods.de2 + self.platform_str = 'Dynamics Explorer 2' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestSES14Methods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.ses14`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['gold'] + self.module = methods.ses14 + self.platform_str = 'Global-scale Observations of the Limb and Disk' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestGPSMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.gps`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['roti15min_jpl'] + self.module = methods.gps + self.platform_str = 'GPS Total Electron Content' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return diff --git a/pysatNASA/version.txt b/pysatNASA/version.txt deleted file mode 100644 index 81340c7e..00000000 --- a/pysatNASA/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.0.4 diff --git a/requirements.txt b/requirements.txt index 5bcb0ece..3684998a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ -netCDF4 -requests beautifulsoup4 -lxml +cdasws cdflib>=0.4.4 +lxml +netCDF4 numpy pandas -pysat>=3.0.4 -xarray<2022.11 +pysat>=3.1.0 +requests +xarray diff --git a/setup.cfg b/setup.cfg index 49912451..08661e10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,55 +1,8 @@ +# name and version must be maintained here as well for python 3.6 compatibility + [metadata] name = pysatNASA -version = file: pysatNASA/version.txt -url = https://github.com/pysat/pysatNASA -author = Jeff Klenzing -author_email = jeffrey.klenzing@nasa.gov -description = 'pysat support for NASA Instruments' -keywords = - pysat - ionosphere -classifiers = - Development Status :: 3 - Alpha - Topic :: Scientific/Engineering :: Physics - Topic :: Scientific/Engineering :: Atmospheric Science - Intended Audience :: Science/Research - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Operating System :: MacOS :: MacOS X - Operating System :: POSIX :: Linux -license_file = LICENSE -long_description = file: README.md -long_description_content_type = text/markdown - -[options] -python_requires = >= 3.5 -setup_requires = - setuptools >= 38.6 - pip >= 10 -include_package_data = True -zip_safe = False -packages = find: -install_requires = - netCDF4 - requests - beautifulsoup4 - lxml - cdflib - numpy - pandas - xarray - pysat - -[options.extras_require] -all = - pysatCDF - -[coverage:report] -omit = - */instruments/templates/ +version = 0.0.5 [flake8] max-line-length = 80 @@ -57,12 +10,3 @@ ignore = D200 D202 W503 - -[tool:pytest] -markers = - all_inst: tests all instruments - download: tests for downloadable instruments - no_download: tests for instruments without download support - load_options: tests for instruments with additional options - first: first tests to run - second: second tests to run diff --git a/setup.py b/setup.py deleted file mode 100644 index 4c0bbd8e..00000000 --- a/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2020, Authors -# Full license can be found in License.md and AUTHORS.md -# ----------------------------------------------------------------------------- -"""Setup routines for pysatNASA. - -Note ----- -Package metadata stored in setup.cfg - -""" - -from setuptools import setup - -# Run setup -setup()