From 9ed8a77045e1f168226621a9879e17d6067aeeec Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:48:57 -0400 Subject: [PATCH 01/17] ENH: added a general clean warning Created a general clean warning variable for the general clean method. --- pysatModels/models/methods/general.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysatModels/models/methods/general.py b/pysatModels/models/methods/general.py index 987bf4a9..bdc6c9fa 100644 --- a/pysatModels/models/methods/general.py +++ b/pysatModels/models/methods/general.py @@ -7,6 +7,10 @@ import pysatModels +clean_warn = {clean_level: [('logger', 'INFO', 'Cleaning not supported', + clean_level)] + for clean_level in ['clean', 'dusty', 'dirty']} + def clean(inst): """Raise a low-level log message about lack of cleaning.""" From bed2a82b6d9fec994c81f6a65e22496923da77c4 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:49:26 -0400 Subject: [PATCH 02/17] ENH: added the `_clean_warn` test attribute Added the `_clean_warn` test attribute to the existing model Instruments. --- pysatModels/models/pydineof_dineof.py | 1 + pysatModels/models/sami2py_sami2.py | 1 + pysatModels/models/ucar_tiegcm.py | 1 + 3 files changed, 3 insertions(+) diff --git a/pysatModels/models/pydineof_dineof.py b/pysatModels/models/pydineof_dineof.py index 23abad97..6d7fc5b9 100644 --- a/pysatModels/models/pydineof_dineof.py +++ b/pysatModels/models/pydineof_dineof.py @@ -69,6 +69,7 @@ _test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}} _test_download = {'': {'': False, 'test': True}} +_clean_warn = {'': {tag: general.clean_warn for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatModels/models/sami2py_sami2.py b/pysatModels/models/sami2py_sami2.py index a31c4733..ad0e6222 100644 --- a/pysatModels/models/sami2py_sami2.py +++ b/pysatModels/models/sami2py_sami2.py @@ -50,6 +50,7 @@ _test_dates = {'': {tag: dt.datetime(2019, 1, 1) for tag in tags.keys()}} _test_download = {'': {'': False, 'test': True}} +_clean_warn = {'': {tag: general.clean_warn for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatModels/models/ucar_tiegcm.py b/pysatModels/models/ucar_tiegcm.py index 43ac1046..c97badfc 100644 --- a/pysatModels/models/ucar_tiegcm.py +++ b/pysatModels/models/ucar_tiegcm.py @@ -54,6 +54,7 @@ _test_dates = {'': {'': dt.datetime(2019, 1, 1), 'icon': dt.datetime(2020, 1, 10)}} _test_download = {'': {'': False, 'icon': True}} +_clean_warn = {'': {tag: general.clean_warn for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From 0cf15ef309d4a09478d836a6de01e77db613d5e0 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:49:39 -0400 Subject: [PATCH 03/17] DOC: updated the changelog Added a description of the changes to the log. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8563e200..eb7e430f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added manual test for pysat and pysatNASA Release Candidates * Added manual test for pysatModels RC pip install * Updated tests to new pysat and pytest standards + * Updated model Instruments to include the new `_clean_warn` attribute * Documentation * Added badges and instructions for PyPi and Zenodo From a8f9ef9fddb7a5a2fa23e83ea6d294d93b308022 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 13:13:45 -0400 Subject: [PATCH 04/17] BUG: pandas compatibility Pandas changed their behaviour at some point, added a try/except loop to ensure the missing time selection works regardless of the available kwargs. --- pysatModels/utils/match.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pysatModels/utils/match.py b/pysatModels/utils/match.py index 2c0534c9..62911711 100644 --- a/pysatModels/utils/match.py +++ b/pysatModels/utils/match.py @@ -162,8 +162,13 @@ def collect_inst_model_pairs(start, stop, tinc, inst, inst_download_kwargs=None, # Download the instrument data, if needed and wanted if not skip_download and (stop - start).days != len(inst.files[start:stop]): - missing_times = [tt for tt in pds.date_range(start, stop, freq='1D', - inclusive='left') + try: + missing_date_range = pds.date_range(start, stop, freq='1D', + inclusive='left') + except TypeError: + missing_date_range = pds.date_range(start, stop, freq='1D') + + missing_times = [tt for tt in missing_date_range if tt not in inst.files[start:stop].index] for tt in missing_times: inst.download(start=tt, stop=tt + pds.DateOffset(days=1), From eeff7dba8dfcd2f02b8f119899793128df599876 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 16:36:03 -0400 Subject: [PATCH 05/17] STY: removed extra whitespace Removed extra whitespace. --- pysatModels/utils/match.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatModels/utils/match.py b/pysatModels/utils/match.py index 62911711..b3542d31 100644 --- a/pysatModels/utils/match.py +++ b/pysatModels/utils/match.py @@ -167,7 +167,7 @@ def collect_inst_model_pairs(start, stop, tinc, inst, inst_download_kwargs=None, inclusive='left') except TypeError: missing_date_range = pds.date_range(start, stop, freq='1D') - + missing_times = [tt for tt in missing_date_range if tt not in inst.files[start:stop].index] for tt in missing_times: From c605be82c55662f49ab46825e397ab8e3d82bb05 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 18:28:11 -0400 Subject: [PATCH 06/17] MAINT: updated pysatNASA behaviour Updated the minimum version of pysatNASA and changed the download behaviour. --- README.md | 14 +++++++------- pysatModels/models/ucar_tiegcm.py | 2 +- requirements.txt | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d57ee54e..724eed14 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ examples on how to use the routines pysatModels uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports Python 3.6+. -| Common modules | Community modules | -| ------------------ | ----------------- | -| numpy | pyForecastTools | -| pandas >= 1.4.0 | pysat >= 3.0.4 | -| requests | pysatNASA | -| scipy | | -| xarray | | +| Common modules | Community modules | +| ------------------ | ------------------ | +| numpy | pyForecastTools | +| pandas >= 1.4.0 | pysat >= 3.0.4 | +| requests | pysatNASA >= 0.0.5 | +| scipy | | +| xarray | | ## Installation through PyPi diff --git a/pysatModels/models/ucar_tiegcm.py b/pysatModels/models/ucar_tiegcm.py index c97badfc..d4186336 100644 --- a/pysatModels/models/ucar_tiegcm.py +++ b/pysatModels/models/ucar_tiegcm.py @@ -253,7 +253,7 @@ def download(date_array, tag, inst_id, data_path=None, **kwargs): temp_dir = tempfile.TemporaryDirectory() # Download using NASA CDAWeb methods in pysatNASA - cdw.download(date_array, tag, inst_id, data_path=temp_dir.name, + cdw.download(date_array, temp_dir.name, tag=tag, inst_id=inst_id, supported_tags=download_tags) # Get a list of files in `temp_dir` diff --git a/requirements.txt b/requirements.txt index 2fc4e0e1..defaf00f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ packaging pandas >= 1.4.0 pyForecastTools pysat >= 3.0.4 -pysatNASA +pysatNASA >= 0.0.5 requests scipy xarray From e72ffbaedb89d06c0d7eb6a260eaabefc9eb95b8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 09:37:32 -0400 Subject: [PATCH 07/17] MAINT: update main.yml Update the tests run, partially complying with new standards. --- .github/workflows/main.yml | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 07e49a05..22a43170 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,12 +13,20 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.9", "3.10"] - numpy_ver: [latest] + python-version: ["3.10", "3.11"] + numpy_ver: ["latest"] + test_config: ["latest"] include: - - python-version: "3.8" - numpy_ver: "1.20.3" - os: "ubuntu-latest" + # NEP29 compliance settings + - python-version: "3.9" + numpy_ver: "1.21" + os: ubuntu-latest + test_config: "NEP29" + # Operational compliance settings + - python-version: "3.6.8" + numpy_ver: "1.19.5" + os: "ubuntu-20.04" + test_config: "Ops" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -29,15 +37,25 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install standard dependencies + - name: Install Operational dependencies + if: ${{ matrix.test_config == 'Ops'}} run: | - pip install -r test_requirements.txt + pip install numpy==${{ matrix.numpy_ver }} pip install -r requirements.txt + pip install -r test_requirements.txt + pip install . - name: Install NEP29 dependencies - if: ${{ matrix.numpy_ver != 'latest'}} + if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} + pip install --upgrade-strategy only-if-needed .[test] + + - name: Install standard dependencies + if: ${{ matrix.test_config == 'latest'}} + run: | + pip install -r test_requirements.txt + pip install -r requirements.txt - name: Set up pysat run: | @@ -51,7 +69,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Run unit and integration tests - run: pytest --cov=pysatModels/ + run: pytest - name: Publish results to coveralls env: From d922ebaff40428730b02caa072c5261754720878 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 09:41:11 -0400 Subject: [PATCH 08/17] MAINT: remove pandas limit Remove the pandas version limit to support earlier versions of python. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index defaf00f..f180b4ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy packaging -pandas >= 1.4.0 +pandas pyForecastTools pysat >= 3.0.4 pysatNASA >= 0.0.5 From 93beb4a0d46aaf7f4599597edabf4e772fd112a2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 09:49:43 -0400 Subject: [PATCH 09/17] MAINT: cycle python version Cycle the supported python versions. --- setup.cfg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 98728f11..df9c3d20 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,16 +14,17 @@ classifiers = License :: Freely Distributable Natural Language :: English Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows license_file = LICENSE [options] -python_requires = >= 3.5 +python_requires = >= 3.6 setup_requires = setuptools >= 38.6; pip >= 10 packages = find: include_package_data = True From c6b2e2dd5fd1da3155e859c2e1441cbe7a606e45 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 16:27:35 -0400 Subject: [PATCH 10/17] BUG: fixed install commands Fixed the commands that expected us to have a pyproject.toml file. --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22a43170..6a1ef829 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,13 +49,16 @@ jobs: if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} - pip install --upgrade-strategy only-if-needed .[test] + pip install --upgrade-strategy only-if-needed -r requirements + pip install --upgrade-strategy only-if-needed -r test_requirements + pip install . - name: Install standard dependencies if: ${{ matrix.test_config == 'latest'}} run: | pip install -r test_requirements.txt pip install -r requirements.txt + pip install . - name: Set up pysat run: | From 2e6a65b085881d0c8279f336acc635916b9fcb6b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 16:57:18 -0400 Subject: [PATCH 11/17] BUG: trying some things Making some hopeful changes. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a1ef829..de855790 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,8 +49,8 @@ jobs: if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} - pip install --upgrade-strategy only-if-needed -r requirements - pip install --upgrade-strategy only-if-needed -r test_requirements + pip install -r requirements + pip install -r test_requirements pip install . - name: Install standard dependencies @@ -72,7 +72,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Run unit and integration tests - run: pytest + run: pytest --cov=pysatModels - name: Publish results to coveralls env: From 58735e93495ba66c84e30897a1a12dfc65ba85fa Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 17:22:54 -0400 Subject: [PATCH 12/17] BUG: add file suffix Forgot to add the file suffixes. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de855790..907b10c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,8 +49,8 @@ jobs: if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} - pip install -r requirements - pip install -r test_requirements + pip install --upgrade-strategy only-if-needed -r requirements.txt + pip install --upgrade-strategy only-if-needed -r test_requirements.txt pip install . - name: Install standard dependencies @@ -72,7 +72,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Run unit and integration tests - run: pytest --cov=pysatModels + run: pytest - name: Publish results to coveralls env: From ec11d1c844c2bc0b30b0457ae806a7483bac95aa Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 7 Jul 2023 17:42:25 -0400 Subject: [PATCH 13/17] BUG: potential fix for coverage drop Coverage dropped to zero, this might fix it. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 907b10c0..75899315 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,7 +72,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Run unit and integration tests - run: pytest + run: pytest --cov=pysatModels/ - name: Publish results to coveralls env: From d380263c05595ecae425f30a14d227a0b0955e2d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 10 Jul 2023 13:09:06 -0400 Subject: [PATCH 14/17] TST: Update .github/workflows/main.yml Update operational test version limits. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75899315..9117b85f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,7 @@ jobs: if: ${{ matrix.test_config == 'Ops'}} run: | pip install numpy==${{ matrix.numpy_ver }} + pip install "cdflib<1.0" pip install -r requirements.txt pip install -r test_requirements.txt pip install . From a3a8b0cddfca1076b11e0f1b1d8bf25bcd0cad96 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 10 Jul 2023 14:15:01 -0400 Subject: [PATCH 15/17] BUG: added a pysatNASA debug line Added a debug line to see if pysatNASA is being installed correctly before trying the pysatModels unit tests. --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75899315..dd29968c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,6 +71,10 @@ jobs: - name: Evaluate complexity run: flake8 . --count --exit-zero --max-complexity=10 --statistics + - name: Install Operational dependencies success + if: ${{ matrix.test_config == 'Ops'}} + run: python -c "import pysatNASA; print(pysatNASA.__version__)" + - name: Run unit and integration tests run: pytest --cov=pysatModels/ From 93f1a5f7336558f57b5854eaf8b0a580f160a8f4 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 10 Jul 2023 14:30:44 -0400 Subject: [PATCH 16/17] BUG: another import Added another import to see if pysatNASA can eventually succeed. --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 11ec2d9a..00c54682 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,9 @@ jobs: - name: Install Operational dependencies success if: ${{ matrix.test_config == 'Ops'}} - run: python -c "import pysatNASA; print(pysatNASA.__version__)" + run: | + pip install pysatNASA==0.0.5 + python -c "import pysatNASA; print(pysatNASA.__version__)" - name: Run unit and integration tests run: pytest --cov=pysatModels/ From c162e4c3c439fc1ac35da370a967cc751080da1f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 10 Jul 2023 14:45:16 -0400 Subject: [PATCH 17/17] BUG: which python See what and where python is. --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 00c54682..ca28a09b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,6 +76,8 @@ jobs: if: ${{ matrix.test_config == 'Ops'}} run: | pip install pysatNASA==0.0.5 + which python + python --version python -c "import pysatNASA; print(pysatNASA.__version__)" - name: Run unit and integration tests