TST: NumPy 2.0 and Python 3.12 (#179)

* BUG: Fix a parsing bug in findsat_mrt
where suffix extraction fails if there are multiple periods in the path.

* TST: NumPy 2.0 and Python 3.12
and also need scipy and scikit-image dev.

* Refactor all the CI and publish workflows to use modern infrastructure.

.github/workflows/ci_workflows.yml

@@ -15,9 +15,6 @@ concurrency:
 permissions:
   contents: read
 
-env:
-  jref: "https://ssb.stsci.edu/trds_open/jref"
-
 jobs:
   initial_checks:
     name: Mandatory checks before CI
@@ -36,94 +33,32 @@ jobs:
             core.info(`PR opened correctly against ${allowed_basebranch}`);
           }
 
-  # The rest only run if above are done
-
-  pep_and_audit:
-    runs-on: ubuntu-latest
-    needs: initial_checks
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Lint with flake8
-      run: |
-        python -m pip install --upgrade pip flake8
-        flake8 acstools --count
-    # Make sure that packaging will work
-    - name: pep517 build
-      run: |
-        python -m pip install --upgrade setuptools build "twine>=3.3"
-        python -m build --sdist .
-        twine check --strict dist/*
-    - name: Security audit
-      run: |
-        python -m pip install --upgrade bandit
-        bandit -r . -c .bandit.yaml
-
   tests:
-    runs-on: ubuntu-latest
     needs: initial_checks
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.9'
-    - name: Install and build
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-        pip install git+https://github.com/spacetelescope/ci_watson.git --upgrade --no-deps
-        python -m pip install -e .[test,all]
-    - name: Test
-      run: pytest --remote-data -v
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
+    with:
+      setenv: |
+        jref: "https://ssb.stsci.edu/trds_open/jref"
+      submodules: false
+      coverage: ''
+      envs: |
+        - name: Lint with flake8
+          linux: codestyle
 
-  dev_deps_tests:
-    runs-on: ubuntu-latest
-    needs: initial_checks
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.11'
-    - name: Install and build
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-        pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps
-        pip install git+https://github.com/spacetelescope/ci_watson.git --upgrade --no-deps
-        python -m pip install -e .[test,all]
-    - name: Test
-      run: pytest --remote-data -v
+        # Make sure that packaging will work
+        - name: pep517 build
+          linux: twine
 
-  link_check:
-    runs-on: ubuntu-latest
-    needs: initial_checks
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Install and build
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-        python -m pip install -e .[docs]
-    - name: Docs link check
-      run: |
-        cd doc
-        make linkcheck
-      shell: bash
+        - name: Security audit
+          linux: bandit
+
+        - name: Check links
+          linux: linkcheck
+
+        - name: Python 3.9 with remote data and all dependencies
+          linux: py39-test-alldeps
+          posargs: --remote-data -v
+
+        - name: Python 3.12 with remote data
+          linux: py312-test-devdeps
+          posargs: --remote-data -v

.github/workflows/open_actions.yml

@@ -17,10 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Label PR
-      uses: actions/labeler@main
+      uses: actions/labeler@v4
       if: github.event_name == 'pull_request_target' || github.event_name == 'pull_request'
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        sync-labels: ''
     - name: 'Comment Draft PR'
       uses: actions/github-script@v6
       if: github.event.pull_request.draft == true
@@ -33,7 +34,7 @@ jobs:
             repo: context.repo.repo,
             body: '👋 Thank you for your draft pull request! Do you know that you can use `[ci skip]` or `[skip ci]` in your commit messages to skip running continuous integration tests until you are ready?'
           })
-    - name: Special comment
-      uses: pllim/action-special_pr_comment@main
-      with:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #- name: Special comment
+    #  uses: pllim/action-special_pr_comment@main
+    #  with:
+    #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/predeps_workflows.yml

@@ -10,32 +10,16 @@ concurrency:
 permissions:
   contents: read
 
-env:
-  jref: "https://ssb.stsci.edu/trds_open/jref"
-
 jobs:
   rc_tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        # TODO: Enable more OSes when possible.
-        #os: [ubuntu-latest, windows-latest, macos-latest]
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-    - name: Install and build
-      run: |
-        python -m pip install --upgrade pip wheel setuptools
-        python -m pip install numpy astropy --pre
-        pip install git+https://github.com/spacetelescope/ci_watson.git --upgrade --no-deps
-        python -m pip install -e .[test,all]
-    - name: Run tests
-      run: pytest --remote-data -v
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
+    with:
+      setenv: |
+        jref: "https://ssb.stsci.edu/trds_open/jref"
+      submodules: false
+      coverage: ''
+    # TODO: Enable more OSes when possible.
+    envs: |
+      - name: Python 3.11 with pre-release
+        linux: py311-test-predeps
+        posargs: --remote-data -v

.github/workflows/publish-to-pypi.yml

@@ -1,16 +1,42 @@
-name: Publish to PyPI
+name: Wheel building
 
 on:
-  release:
-    types: [released]
+  pull_request:
+    # We also want this workflow triggered if the 'Build all wheels'
+    # label is added or present when PR is updated
+    types:
+      - synchronize
+      - labeled
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+
+permissions:
+  contents: read
 
 jobs:
-  publish:
-    uses: spacetelescope/action-publish_to_pypi/.github/workflows/workflow.yml@master
+  build_and_publish:
+    # This job builds the wheels and publishes them to PyPI for all
+    # tags. For PRs with the "Build wheels" label, wheels are built,
+    # but are not uploaded to PyPI.
+
+    permissions:
+      contents: none
+
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1
+
+    if: (github.repository == 'spacetelescope/acstools' && (github.event_name == 'push' ||  contains(github.event.pull_request.labels.*.name, 'Build wheels')))
     with:
-      test: false
-      build_platform_wheels: false # Set to true if your package contains a C extension
+      env: |
+        jref: "https://ssb.stsci.edu/trds_open/jref"
+
+      # We upload to PyPI for all tag pushes
+      upload_to_pypi: ${{ startsWith(github.ref, 'refs/tags/') && github.event_name == 'push' }}
+
+      test_extras: test,all
+      test_command: pytest -p no:warnings --pyargs acstools.tests --remote-data -v
+
     secrets:
-      user: ${{ secrets.PYPI_USERNAME_STSCI_MAINTAINER }}
-      password: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }} # WARNING: Do not hardcode secret values here! If you want to use a different user or password, you can override this secret by creating one with the same name in your Github repository settings.
-      test_password: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER_TEST }}
+      pypi_token: ${{ secrets.PYPI_TOKEN }}

acstools/findsat_mrt.py

@@ -1250,7 +1250,7 @@ def __init__(self,
                 raise ValueError('This program does not yet work on subarrays')
 
             # get suffix to determine how to process image
-            suffix = (self.image_file.split('.')[0]).split('_')[-1].lower()
+            suffix = (self.image_file.split('.')[-2]).split('_')[-1].lower()
             self.image_type = suffix
             LOG.info('image type is {}'.format(self.image_type))
 

pyproject.toml

@@ -1,5 +1,4 @@
 [build-system]
 requires = ["setuptools>=30.3.0",
-            "setuptools_scm",
-            "wheel"]
+            "setuptools_scm"]
 build-backend = "setuptools.build_meta"

setup.cfg

@@ -7,7 +7,7 @@ keywords = astronomy, astrophysics, calibration
 author = Matt Davis, Warren Hack, Norman Grogin, Pey Lian Lim, Sara Ogaz, Leonardo Ubeda, Mihai Cara, David Borncamp, Nathan Miles, Tyler Desjardins, Jenna Ryon, David Stark
 author_email = [email protected]
 license = BSD
-license_file = LICENSE.md
+license_files = LICENSE.md
 url = https://github.com/spacetelescope/acstools
 edit_on_github = False
 github_project = spacetelescope/acstools
@@ -67,7 +67,6 @@ console_scripts =
   acs_destripe_plus = acstools.acs_destripe_plus:main
 
 [tool:pytest]
-addopts = --doctest-ignore-import-errors
 minversion = 5
 testpaths = "acstools" "doc"
 norecursedirs = build doc/build
@@ -79,11 +78,11 @@ filterwarnings =
   ignore:unclosed file:ResourceWarning
   ignore:distutils Version classes are deprecated:DeprecationWarning
   ignore:Deprecation Warning.*timeout.*parameter no longer needed:UserWarning
-  # This is from skimage import
-  ignore:.*np\.bool8.*is a deprecated alias:DeprecationWarning
   # https://github.com/spacetelescope/crds/issues/922
   ignore:Deprecated call to.*pkg_resources\.declare_namespace.*:DeprecationWarning
   ignore:pkg_resources is deprecated as an API:DeprecationWarning
+  # Python 3.12 warning from dateutil imported by matplotlib
+  ignore:.*utcfromtimestamp:DeprecationWarning
 
 [flake8]
 # Ignoring these for now:

tox.ini

@@ -0,0 +1,96 @@
+[tox]
+envlist =
+    py{38,39,310,311,312}-test{,-alldeps,-devdeps,-predeps}
+    codestyle
+    twine
+    bandit
+    linkcheck
+
+[testenv]
+# Pass through the following environment variables which are needed for the CI
+passenv = HOME,CI,jref
+
+setenv =
+    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+
+# Run the tests in a temporary directory to make sure that we don't import
+# package from the source tree
+changedir = .tmp/{envname}
+
+# tox environments are constructued with so-called 'factors' (or terms)
+# separated by hyphens, e.g. test-devdeps. Lines below starting with factor:
+# will only take effect if that factor is included in the environment name. To
+# see a list of example environments that can be run, along with a description,
+# run:
+#
+#     tox -l -v
+#
+description =
+    run tests
+    alldeps: with all optional dependencies
+    devdeps: with the latest developer version of key dependencies
+    predeps: with the pre-release version of key dependencies
+
+deps =
+    # The devdeps factor is intended to be used to install the latest developer version
+    # or nightly wheel of key dependencies.
+    devdeps: numpy>=0.0.dev0
+    devdeps: scipy>=0.0.dev0
+    devdeps: matplotlib>=0.0.dev0
+    devdeps: scikit-image>=0.0.dev0
+    devdeps: astropy>=0.0.dev0
+    devdeps: git+https://github.com/astropy/photutils.git
+    devdeps: git+https://github.com/spacetelescope/ci_watson.git
+    devdeps: stsci.tools
+
+extras =
+    test
+    alldeps: all
+    predeps: all
+
+commands =
+    # Force numpy-dev after matplotlib downgrades it (https://github.com/matplotlib/matplotlib/issues/26847)
+    devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
+    pip freeze
+    pytest --pyargs acstools {toxinidir}/doc {posargs}
+
+pip_pre =
+    predeps: true
+    !predeps: false
+
+[testenv:codestyle]
+skip_install = true
+changedir = {toxinidir}
+description = check code style with flake8
+deps = flake8
+commands = flake8 acstools --count
+
+[testenv:twine]
+skip_install = true
+changedir = {toxinidir}
+description = twine check dist tarball
+deps =
+    build
+    twine>=3.3
+commands =
+    pip freeze
+    python -m build --sdist .
+    twine check --strict dist/*
+
+[testenv:bandit]
+skip_install = true
+changedir = {toxinidir}
+description = Security audit with bandit
+deps = bandit
+commands =
+    pip freeze
+    bandit -r acstools -c .bandit.yaml
+
+[testenv:linkcheck]
+changedir = doc
+description = check the links in the HTML docs
+extras = docs
+allowlist_externals = make
+commands =
+    pip freeze
+    make linkcheck