diff --git a/.bandit.yaml b/.bandit.yaml new file mode 100644 index 00000000..c895a6d9 --- /dev/null +++ b/.bandit.yaml @@ -0,0 +1,2 @@ +exclude_dirs: + - acstools/tests diff --git a/.gitignore b/.gitignore index d2ad55e9..2d43cee7 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,8 @@ sdist develop-eggs .installed.cfg distribute-*.tar.gz +pip-wheel-metadata/ +acstools/version.py # Other .cache diff --git a/.readthedocs.yml b/.readthedocs.yml index 4d500323..469bcc4d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,9 +12,10 @@ python: version: 3.7 system_packages: true install: - - requirements: doc/rtd-pip-requirements - - method: setuptools + - method: pip path: . + extra_requirements: + - docs # Don't build any extra formats formats: [] diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 00000000..f1d59ac8 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1 @@ +See https://github.com/spacetelescope/acstools/releases diff --git a/Jenkinsfile b/Jenkinsfile index 23d3ada2..67db4c68 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,25 +8,18 @@ bc1.name = "release" // Would be nice if Jenkins can access /grp/hst/cdbs/xxxx directly. bc1.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory'] bc1.conda_channels = ['http://ssb.stsci.edu/astroconda'] -bc1.conda_packages = ['python=3.6', - 'requests', - 'numpy', - 'matplotlib', - 'scipy', - 'stsci.tools'] -bc1.build_cmds = ["pip install scikit-image ci-watson", - "python setup.py install"] +bc1.conda_packages = ['python=3.6'] +bc1.build_cmds = ["pip install -e .[test,all]"] bc1.test_cmds = ["pytest --basetemp=tests_output --junitxml results.xml --bigdata -v"] bc1.failedUnstableThresh = 1 bc1.failedFailureThresh = 6 -// Run with astropy dev and Python 3.7 +// Run with astropy dev and Python 3.8 bc2 = utils.copy(bc1) bc2.name = "dev" -bc2.conda_packages[0] = "python=3.7" -bc2.build_cmds = ["pip install scikit-image ci-watson", - "pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps", - "python setup.py install"] +bc2.conda_packages[0] = "python=3.8" +bc2.build_cmds = ["pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps", + "pip install -e [test,all]"] // Iterate over configurations that define the (distibuted) build matrix. // Spawn a host of the given nodetype for each combination and run in parallel. diff --git a/JenkinsfileRT b/JenkinsfileRT index bcaf8d8b..7c8d094a 100644 --- a/JenkinsfileRT +++ b/JenkinsfileRT @@ -14,25 +14,18 @@ bc.name = "release" bc.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory', 'jref=/grp/hst/cdbs/jref/'] bc.conda_channels = ['http://ssb.stsci.edu/astroconda'] -bc.conda_packages = ['python=3.6', - 'requests', - 'numpy', - 'matplotlib', - 'scipy', - 'stsci.tools'] -bc.build_cmds = ["pip install scikit-image ci-watson", - "python setup.py install"] +bc.conda_packages = ['python=3.6'] +bc.build_cmds = ["pip install -e .[test,all]"] bc.test_cmds = ["pytest --basetemp=tests_output --junitxml results.xml --bigdata --slow -v"] bc.failedUnstableThresh = 1 bc.failedFailureThresh = 6 -// Astropy dev and Python 3.7 +// Astropy dev and Python 3.8 bc1 = utils.copy(bc) bc1.name = "dev" -bc1.conda_packages[0] = "python=3.7" -bc1.build_cmds = ["pip install scikit-image", - "pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps", - "python setup.py install"] +bc1.conda_packages[0] = "python=3.8" +bc1.build_cmds = ["pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps", + "pip install -e [test,all]"] // Iterate over configurations that define the (distributed) build matrix. // Spawn a host (or workdir) for each combination and run in parallel. diff --git a/LICENSE.md b/LICENSE.md index 808bee42..11e7b7d9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2018, Space Telescope Science Institute, AURA +Copyright (c) 2020, Space Telescope Science Institute, AURA All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/MANIFEST.in b/MANIFEST.in index b34265be..69366c56 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,8 @@ include README.rst include LICENSE.txt +include CHANGES.rst include setup.cfg +include pyproject.toml recursive-include doc * prune build diff --git a/acstools/__init__.py b/acstools/__init__.py index 25baeea7..27f253d3 100644 --- a/acstools/__init__.py +++ b/acstools/__init__.py @@ -1,31 +1,21 @@ -"""The acstools package holds Python tasks useful for analyzing ACS data. - -These tasks include: - -Utility and library functions used by these tasks are also included in this -module. - -""" -from pkg_resources import get_distribution, DistributionNotFound - - +"""The acstools package holds Python tasks useful for analyzing ACS data.""" try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: + from .version import version as __version__ +except ImportError: # package is not installed - __version__ = 'unknown' + __version__ = '' -from . import acs_destripe -from . import acs_destripe_plus -from . import calacs -from . import acsccd -from . import acscte -from . import acscteforwardmodel -from . import acs2d -from . import acsrej -from . import acssum -from . import acszpt -from . import acsphotcte -from . import satdet -from . import utils_calib +from . import acs_destripe # noqa +from . import acs_destripe_plus # noqa +from . import calacs # noqa +from . import acsccd # noqa +from . import acscte # noqa +from . import acscteforwardmodel # noqa +from . import acs2d # noqa +from . import acsrej # noqa +from . import acssum # noqa +from . import acszpt # noqa +from . import acsphotcte # noqa +from . import satdet # noqa +from . import utils_calib # noqa diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 36fd3277..dbe97b56 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,18 +5,19 @@ trigger: - master jobs: -- job: 'EggInfo' +- job: 'PEP517' pool: vmImage: 'Ubuntu-16.04' steps: - task: UsePythonVersion@0 - # Make sure that egg_info works without dependencies + # Make sure that packaging will work - script: | - python -m pip install --upgrade pip setuptools - python setup.py egg_info - displayName: 'egg_info' + python -m pip install --upgrade pip setuptools pep517 twine + python -m pep517.build --source . + twine check dist/* + displayName: 'pep517 build' - job: 'PEP8' pool: @@ -32,3 +33,36 @@ jobs: pip install flake8 flake8 acstools --count displayName: 'PEP 8 check' + +- job: 'Audit' + pool: + vmImage: 'Ubuntu-16.04' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + + - script: | + python -m pip install --upgrade pip setuptools + pip install bandit + bandit -r . -c .bandit.yaml + displayName: 'Security audit' + +- job: 'LinkCheck' + pool: + vmImage: 'Ubuntu-16.04' + strategy: + maxParallel: 4 + + steps: + - task: UsePythonVersion@0 + + - script: | + python -m pip install --upgrade pip setuptools + pip install -e .[docs] + + - bash: | + cd doc + make linkcheck + displayName: 'Run docs link check' diff --git a/conftest.py b/conftest.py index d884a1d6..90367149 100644 --- a/conftest.py +++ b/conftest.py @@ -1,34 +1,31 @@ """Custom ``pytest`` configurations.""" +try: + from pytest_astropy_header.display import (PYTEST_HEADER_MODULES, + TESTED_VERSIONS) +except ImportError: + PYTEST_HEADER_MODULES = {} + TESTED_VERSIONS = {} -from astropy.tests.helper import enable_deprecations_as_exceptions +try: + from acstools.version import version +except ImportError: + version = 'unknown' -# Turn deprecation warnings into exceptions. +# Uncomment the following line to treat all DeprecationWarnings as +# exceptions # NOTE: socks warning fixed by https://github.com/Anorov/PySocks/pull/106 # but not released yet. +from astropy.tests.helper import enable_deprecations_as_exceptions enable_deprecations_as_exceptions(warnings_to_ignore_entire_module=['socks']) +# Uncomment and customize the following lines to add/remove entries +# from the list of packages for which version numbers are displayed +# when running the tests. +PYTEST_HEADER_MODULES['Astropy'] = 'astropy' +PYTEST_HEADER_MODULES['beautifulsoup4'] = 'bs4' +PYTEST_HEADER_MODULES['requests'] = 'requests' +PYTEST_HEADER_MODULES['stsci.tools'] = 'stsci.tools' +PYTEST_HEADER_MODULES.pop('Pandas') +PYTEST_HEADER_MODULES.pop('h5py') -# For easy inspection on what dependencies were used in test. -def pytest_report_header(config): - import sys - import warnings - from astropy.utils.introspection import resolve_name - - s = "\nFull Python Version: \n{0}\n\n".format(sys.version) - - for module_name in ('numpy', 'astropy', 'scipy', 'matplotlib', - 'stsci.tools'): - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - module = resolve_name(module_name) - except ImportError: - s += "{0}: not available\n".format(module_name) - else: - try: - version = module.__version__ - except AttributeError: - version = 'unknown (no __version__ attribute)' - s += "{0}: {1}\n".format(module_name, version) - - return s +TESTED_VERSIONS['acstools'] = version diff --git a/doc/rtd-pip-requirements b/doc/rtd-pip-requirements deleted file mode 100644 index 29da547f..00000000 --- a/doc/rtd-pip-requirements +++ /dev/null @@ -1,3 +0,0 @@ -numpy>=1.13 -astropy -sphinx-automodapi diff --git a/doc/source/conf.py b/doc/source/conf.py index b8ed5d8c..c2f6fa8f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -205,10 +205,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'python': ('http://docs.python.org/', None), - 'numpy': ('http://docs.scipy.org/doc/numpy/', None), - 'scipy': ('http://docs.scipy.org/doc/scipy/reference/', None), - 'skimage': ('http://scikit-image.org/docs/0.11.x/', None), - 'matplotlib': ('http://matplotlib.org/', None), - 'astropy': ('http://docs.astropy.org/en/stable/', None) + 'python': ('https://docs.python.org/', None), + 'numpy': ('https://docs.scipy.org/doc/numpy/', None), + 'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None), + 'skimage': ('https://scikit-image.org/docs/0.11.x/', None), + 'matplotlib': ('https://matplotlib.org/', None), + 'astropy': ('https://docs.astropy.org/en/stable/', None) } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..46a745ed --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +requires = ["setuptools>=30.3.0", + "setuptools_scm", + "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index da93dfe9..e1e91f96 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,26 +1,62 @@ [metadata] -package_name = acstools -description = Python Tools for ACS (Advanced Camera for Surveys) Data +name = acstools +description = Python Tools for HST ACS +long_description = Python Tools for HST ACS (Advanced Camera for Surveys) Data +long_description_content_type = text/plain +keywords = astronomy, astrophysics, calibration author = Matt Davis, Warren Hack, Norman Grogin, Pey Lian Lim, Sara Ogaz, Leornado Ubeda, Mihai Cara, David Borncamp, Nathan Miles author_email = help@stsci.edu license = BSD +license_file = LICENSE.md url = https://github.com/spacetelescope/acstools +edit_on_github = False +github_project = spacetelescope/acstools classifier = Intended Audience :: Science/Research License :: OSI Approved :: BSD License Operating System :: OS Independent Programming Language :: Python :: 3 + Programming Language :: Python :: Implementation :: CPython Topic :: Scientific/Engineering :: Astronomy Topic :: Scientific/Engineering :: Physics Topic :: Software Development :: Libraries :: Python Modules +[options] +packages = find: +zip_safe = False +setup_requires= + setuptools_scm +install_requires = + numpy + astropy>=2 + beautifulsoup4 + requests +python_requires = >=3.5 + +[options.extras_require] +all = + matplotlib + scipy + scikit-image + stsci.tools + stsci.imagestats +test = + pytest + pytest-astropy-header + ci-watson +docs = + sphinx-automodapi + [entry_points] acs_destripe = acstools.acs_destripe:main acs_destripe_plus = acstools.acs_destripe_plus:main [tool:pytest] -minversion = 3.0 +minversion = 4.0 +testpaths = "acstools" "doc" norecursedirs = build doc/build +astropy_header = true +xfail_strict = true junit_family=xunit2 [flake8] @@ -33,4 +69,3 @@ junit_family=xunit2 # E704: multiple statements on one line (def) # W504: line break after binary operator ignore = E221,E226,E262,E265,E501,E704,W504 -exclude = __init__.py diff --git a/setup.py b/setup.py index d8ae2218..2efe7310 100755 --- a/setup.py +++ b/setup.py @@ -1,47 +1,37 @@ #!/usr/bin/env python +import sys from setuptools import setup -# Get some values from the setup.cfg -from configparser import ConfigParser -conf = ConfigParser() -conf.read(['setup.cfg']) -metadata = dict(conf.items('metadata')) - -PACKAGENAME = metadata.get('package_name', 'packagename') -DESCRIPTION = metadata.get('description', 'package') -AUTHOR = metadata.get('author', '') -AUTHOR_EMAIL = metadata.get('author_email', '') -LICENSE = metadata.get('license', 'unknown') -URL = metadata.get('url', 'http://www.stsci.edu') -CLASSIFIERS = [c for c in metadata.get('classifier', ['']).splitlines() if c] - -# Define entry points for command-line scripts -entry_points = {'console_scripts': []} -entry_point_list = conf.items('entry_points') -for entry_point in entry_point_list: - entry_points['console_scripts'].append('{0} = {1}'.format(entry_point[0], - entry_point[1])) -setup( - name=PACKAGENAME, - use_scm_version=True, - description=DESCRIPTION, - author=AUTHOR, - author_email=AUTHOR_EMAIL, - license=LICENSE, - url=URL, - classifiers=CLASSIFIERS, - packages=[PACKAGENAME], - package_dir={PACKAGENAME: PACKAGENAME}, - package_data={PACKAGENAME: ['pars/*']}, - entry_points=entry_points, - python_requires='>=3.5', - setup_requires=['setuptools_scm'], - install_requires=[ - 'astropy>=2', - 'numpy', - 'beautifulsoup4', - 'requests' - ], - tests_require=['pytest'], - zip_safe=False -) +TEST_HELP = """ +Note: running tests is no longer done using 'python setup.py test'. Instead +you will need to run: + + pip install -e . + pytest + +For more information, see: + + https://docs.astropy.org/en/latest/development/testguide.html#running-tests +""" + +if 'test' in sys.argv: + print(TEST_HELP) + sys.exit(1) + +DOCS_HELP = """ +Note: building the documentation is no longer done using +'python setup.py build_docs'. Instead you will need to run: + + cd docs + make html + +For more information, see: + + https://docs.astropy.org/en/latest/install.html#builddocs +""" + +if 'build_docs' in sys.argv or 'build_sphinx' in sys.argv: + print(DOCS_HELP) + sys.exit(1) + +setup(use_scm_version={'write_to': 'acstools/version.py'})