From e65c7298a0e3b3f784ec5aff875a8c44059438aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Kope=C4=87?= <3338226+mkopec87@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:19:48 +0100 Subject: [PATCH 1/5] * Switch from setup.py to pyproject.toml * Add numpy<2,pandas<2 test environment to build pipeline test matrix * Refactor build pipeline config file --- .github/workflows/test.yml | 52 ++++++++------ CHANGES.rst | 3 + pyproject.toml | 72 +++++++++++++++++++ requirements-test.txt | 7 -- requirements.txt | 3 - setup.py | 137 ------------------------------------- 6 files changed, 106 insertions(+), 168 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements-test.txt delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e10c0e..e77170d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,27 +9,37 @@ jobs: test: strategy: matrix: - os: [ubuntu-latest] - python: ['3.9', '3.10', '3.11', '3.12'] + os: [ ubuntu-latest ] + python: [ "3.9", "3.10", "3.11", "3.12" ] + numpy_version: [ "numpy-latest", "numpy<2" ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python }} - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-test.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -r requirements-test.txt - - name: Test with pytest - run: | - pytest tests/ + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python }} + + - name: Use cache for pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles("**/pyproject.toml") }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ "${{ matrix.numpy_version }}" = "numpy<2" ]; then + pip install ".[test,test_numpy_pre2]" + else + pip install ".[test]" + fi + + - name: Test with pytest + run: | + pytest tests diff --git a/CHANGES.rst b/CHANGES.rst index 3dc6c51..73dc147 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,9 @@ Version 1.0.34, Dec 2024 * Remove unused test_gpu.twosigfigs function. * Refactor tests with Numpy() and Pandas() context managers to use single 'with' statement. +* Switch from setup.py to pyproject.toml +* Add numpy<2,pandas<2 test environment to build pipeline test matrix + Version 1.0.33, Dec 2022 ------------------------ * fix of get_sub_hist() when Bin histogram is filled only with nans. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..51f729f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,72 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "histogrammar" +description = "Composable histogram primitives for distributed data reduction" +keywords = [ + "pandas", + "spark", + "data-science", + "data-analysis", + "statistics", + "python", + "jupyter", + "ipython" +] +readme = "README.rst" +requires-python = ">=3.9" +authors = [{ name = "Jim Pivarski (DIANA-HEP)", email = "pivarski@fnal.gov" }, { name = "Max Baak", email = "maxbaak@gmail.com" }] +maintainers = [{ name = "Max Baak", email = "maxbaak@gmail.com" }] +license = { type = "Apache Software License v2", file = "LICENSE" } +dependencies = [ + "numpy", + "tqdm", + "joblib>=0.14.0" +] +classifiers = ["Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Physics", +] +dynamic = ["version"] + +[project.optional-dependencies] +test = [ + "ipykernel>=5.1.3", + "jupyter_client>=5.2.3", + "matplotlib", + "pandas", + "pre-commit>=2.9.0", + "pytest-notebook>=0.6.1", + "pytest>=4.0.2", +] +test_numpy_pre2 = [ + "numpy<2", + "pandas<2", +] + +# files to be shipped with the installation, under: histogrammar/test_data and histogrammar/notebooks +# after installation, these can be found with the functions in resources.py +[tool.setuptools.package-data] +histogrammar = [ + "test_data/*.csv.gz", + "test_data/*.json*", + "notebooks/*tutorial*.ipynb", +] + +[project.urls] +repository = "https://github.com/histogrammar/histogrammar-python" + +[tool.semantic_release] +version_variable = [ + "histogrammar/version.py:version", +] +build_command = "pip install build && python -m build" + +[tool.setuptools.dynamic] +version = { attr = "histogrammar.version.version" } diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 50075d3..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,7 +0,0 @@ -pytest>=4.0.2 -pytest-notebook>=0.6.1 -jupyter_client>=5.2.3 -ipykernel>=5.1.3 -pre-commit>=2.9.0 -matplotlib -pandas diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3a21ef3..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy -tqdm -joblib>=0.14.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3fbaa3a..0000000 --- a/setup.py +++ /dev/null @@ -1,137 +0,0 @@ -# flake8: noqa - -#!/usr/bin/env python - -# Copyright 2016 DIANA-HEP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup, find_packages - -NAME = "histogrammar" - -MAJOR = 1 -REVISION = 0 -PATCH = 34 -DEV = False -# NOTE: also update version at: README.rst and update CHANGES.rst - -with open("requirements.txt") as f: - REQUIREMENTS = f.read().splitlines() - -VERSION = "{major}.{revision}.{patch}".format( - major=MAJOR, revision=REVISION, patch=PATCH -) -FULL_VERSION = VERSION -if DEV: - FULL_VERSION += ".dev" - with open("requirements-test.txt") as f: - REQUIREMENTS += f.read().splitlines() - -# read the contents of abstract file -with open("README.rst", encoding="utf-8") as f: - long_description = f.read() - - -def write_version_py(filename: str = "histogrammar/version.py") -> None: - """Write package version to version.py. - - This will ensure that the version in version.py is in sync with us. - - :param filename: The version.py to write too. - :type filename: str - """ - # Do not modify the indentation of version_str! - version_str = """\"\"\"THIS FILE IS AUTO-GENERATED BY SETUP.PY.\"\"\" - -import re - -name = \"{name!s}\" -__version__ = \"{version!s}\" -version = \"{version!s}\" -full_version = \"{full_version!s}\" -release = {is_release!s} - -version_info = tuple(re.split(r"[-\.]", __version__)) -specification = ".".join(version_info[:2]) - - -def compatible(serializedVersion): - selfMajor, selfMinor = map(int, version_info[:2]) - otherMajor, otherMinor = map(int, re.split(r"[-\.]", serializedVersion)[:2]) - if selfMajor >= otherMajor: - return True - elif selfMinor >= otherMinor: - return True - else: - return False -""" - - with open(filename, "w") as version_file: - version_file.write( - version_str.format( - name=NAME.lower(), - version=VERSION, - full_version=FULL_VERSION, - is_release=not DEV, - ) - ) - - -def setup_package() -> None: - """The main setup method. - - It is responsible for setting up and installing the package. - """ - write_version_py() - - setup(name=NAME, - version=VERSION, - packages=find_packages(), - scripts=["scripts/hgwatch"], - description="Composable histogram primitives for distributed data reduction.", - long_description=long_description, - long_description_content_type="text/x-rst", - python_requires=">=3.6", - author="Jim Pivarski (DIANA-HEP)", - author_email="pivarski@fnal.gov", - maintainer="Max Baak", - maintainer_email="maxbaak@gmail.com", - url="https://histogrammar.github.io/histogrammar-docs", - download_url="https://github.com/histogrammar/histogrammar-python", - license="Apache Software License v2", - test_suite="tests", - install_requires=REQUIREMENTS, - classifiers=["Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Scientific/Engineering :: Physics", - ], - # files to be shipped with the installation, under: popmon/popmon/ - # after installation, these can be found with the functions in resources.py - package_data=dict( - histogrammar=[ - "test_data/*.csv.gz", - "test_data/*.json*", - "notebooks/*tutorial*.ipynb", - ] - ), - platforms="Any", - ) - - -if __name__ == "__main__": - setup_package() From 4a2f873528fe10ceb9596c328c38c6bc44e3ba9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Kope=C4=87?= <3338226+mkopec87@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:23:26 +0100 Subject: [PATCH 2/5] * Fix quotes in build config cache key --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e77170d..2d90652 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles("**/pyproject.toml") }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-pip- From 4b4f379fb99c22d884e4ea4373f5b45f2aaeb761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Kope=C4=87?= <3338226+mkopec87@users.noreply.github.com> Date: Sun, 15 Dec 2024 07:33:41 +0100 Subject: [PATCH 3/5] * Add spark tests * Add pandas optional dependency --- .github/workflows/test.yml | 2 +- pyproject.toml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d90652..966d87b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: if [ "${{ matrix.numpy_version }}" = "numpy<2" ]; then pip install ".[test,test_numpy_pre2]" else - pip install ".[test]" + pip install ".[test,spark]" fi - name: Test with pytest diff --git a/pyproject.toml b/pyproject.toml index 51f729f..e70936a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,12 @@ classifiers = ["Development Status :: 5 - Production/Stable", dynamic = ["version"] [project.optional-dependencies] +pandas = [ + "pandas" +] +spark = [ + "pyspark>=3.1; python_version <= '3.11'", +] test = [ "ipykernel>=5.1.3", "jupyter_client>=5.2.3", From b1f8f7f3006a30c7590423d215e6c58daca4f8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Kope=C4=87?= <3338226+mkopec87@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:52:22 +0100 Subject: [PATCH 4/5] * Refactor version.py --- histogrammar/version.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/histogrammar/version.py b/histogrammar/version.py index bb60b61..0648a0c 100644 --- a/histogrammar/version.py +++ b/histogrammar/version.py @@ -1,23 +1,24 @@ -"""THIS FILE IS AUTO-GENERATED BY SETUP.PY.""" - import re +from typing import Tuple -name = "histogrammar" -__version__ = "1.0.34" version = "1.0.34" -full_version = "1.0.34" -release = True -version_info = tuple(re.split(r"[-\.]", __version__)) -specification = ".".join(version_info[:2]) +def split_version_string(version_string: str) -> Tuple[int, int]: + version_numbers = list(map(int, re.split(r"[-.]", version_string))) + return version_numbers[0], version_numbers[1] + + +specification = ".".join([str(i) for i in split_version_string(version)[:2]]) + + +def compatible(serialized_version: str) -> bool: + self_major, self_minor = split_version_string(version) + other_major, other_minor = split_version_string(serialized_version) -def compatible(serializedVersion): - selfMajor, selfMinor = map(int, version_info[:2]) - otherMajor, otherMinor = map(int, re.split(r"[-\.]", serializedVersion)[:2]) - if selfMajor >= otherMajor: + if self_major >= other_major: return True - elif selfMinor >= otherMinor: + elif self_minor >= other_minor: return True else: return False From fb20090a1e5d2fb030691c29fbdf2a2fecce9ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Kope=C4=87?= <3338226+mkopec87@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:03:59 +0100 Subject: [PATCH 5/5] * Test spark with numpy<2 in build matrix configuration --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 966d87b..4ec2a0a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,9 +35,9 @@ jobs: run: | python -m pip install --upgrade pip if [ "${{ matrix.numpy_version }}" = "numpy<2" ]; then - pip install ".[test,test_numpy_pre2]" + pip install ".[test,pandas,spark,test_numpy_pre2]" else - pip install ".[test,spark]" + pip install ".[test,pandas,spark]" fi - name: Test with pytest