From b970a2e5f3f63f264c9ccbd8b0757f01697f3efc Mon Sep 17 00:00:00 2001 From: Scott-Simmons <52365471+Scott-Simmons@users.noreply.github.com> Date: Wed, 20 Nov 2024 07:01:46 +1100 Subject: [PATCH] Dockerise testing for dev (#1093) * Add a dockerfile for testing Add image for testing tsfresh with multiple python versions. This can be a good final sanity check e.g. before a release. * Nit: add document start on yaml file * Initialise makefile Targets include 1. Building dockerised testing environment 2. Testing tsfresh within testing environment 3. Testing tsfresh locally with tox with available python versions 4. Testing tsfresh locally with current active python version 5. Cleaning build artifacts 6. Formatting. This is so that changes can be made before pushing the changes up the remote to trigger the formatting checks. TODO: pre-commit hooks should probs be added too. * Documentation updates Change docs to reflect the new testing invokations. * Responded to PR review - Remove the python PATCH version from makefile - Move calls to `make clean` to be part of the docker test invokation instead of within local test commands - Removed the one line makefile commands (the docs have enough info on the development process so will not bother using `make` for these. - Updated hyperlink to point to main instead of specific commit sha --- .pre-commit-config.yaml | 1 + Dockerfile.testing | 62 +++++++++++++++++++++++++++++++++ Makefile | 31 +++++++++++++++++ docs/text/how_to_contribute.rst | 15 ++++++-- 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.testing create mode 100644 Makefile diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d3b6089..d0754db1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +--- repos: - repo: https://github.com/psf/black rev: 22.12.0 diff --git a/Dockerfile.testing b/Dockerfile.testing new file mode 100644 index 00000000..4641ea3c --- /dev/null +++ b/Dockerfile.testing @@ -0,0 +1,62 @@ +# Bakes the python versions which tsfresh targets into a testing env +FROM ubuntu:22.04 + +SHELL ["/bin/bash", "-c"] + +# These are required to build python from source +RUN apt-get update && apt-get install -y \ + python3-pip \ + curl \ + clang \ + git \ + build-essential \ + libssl-dev \ + libreadline-dev \ + zlib1g-dev \ + libbz2-dev \ + libsqlite3-dev \ + llvm \ + libncurses5-dev \ + libgdbm-dev \ + libnss3-dev \ + libffi-dev \ + liblzma-dev \ + libgmp-dev \ + libmpfr-dev \ + && apt-get clean + + +RUN curl https://pyenv.run | bash + +# For interactive use (if any), this is an edge case. +RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \ + echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \ + echo 'eval "$(pyenv init -)"' >> ~/.bashrc + +ENV PYENV_ROOT="/root/.pyenv" +ENV PATH="$PYENV_ROOT/bin:$PATH" +ENV PATH="$PYENV_ROOT/shims:$PATH" + +ARG PYTHON_VERSIONS +RUN for version in $PYTHON_VERSIONS; do \ + echo Installing $version; \ + # Band aid for https://github.com/pyenv/pyenv/issues/1738 + # since this also appears to apply to 3.7.X + if [[ $version =~ ^3\.7\..*$ ]]; then \ + echo Using clang to compile $version; \ + CC=clang pyenv install $version || exit 1; \ + else \ + pyenv install $version || exit 1; \ + fi; \ + done + +RUN pyenv global $PYTHON_VERSIONS + +RUN pip install tox + +WORKDIR /tsfresh + +# Requires adding safe.directory so that tsfresh can build when the +# repo is mounted. +# Note cannot do this at build time as no git directory exists +CMD ["/bin/bash", "-c", "git config --global --add safe.directory /tsfresh && tox -r -p auto"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..ce6c3f55 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +WORKDIR := /tsfresh +TEST_IMAGE := tsfresh-test-image +TEST_DOCKERFILE := Dockerfile.testing +TEST_CONTAINER := tsfresh-test-container +PYTHON_VERSIONS := "3.7 3.8 3.9 3.10 3.11" + +# Tests `PYTHON_VERSIONS`, provided they are also +# specified in setup.cfg `envlist` +test-all-testenv: build-docker-testenv run-docker-tests clean + +build-docker-testenv: + docker build \ + -f $(TEST_DOCKERFILE) \ + -t $(TEST_IMAGE) \ + --build-arg PYTHON_VERSIONS=$(PYTHON_VERSIONS) \ + . + +run-docker-tests: + docker run --rm \ + --name $(TEST_CONTAINER) \ + -v .:$(WORKDIR) \ + -v build_artifacts:$(WORKDIR)/build \ + -v tox_artifacts:$(WORKDIR)/.tox \ + -v egg_artifacts:$(WORKDIR)/tsfresh.egg-info \ + $(TEST_IMAGE) + +clean: + rm -rf .tox build/ dist/ *.egg-info + + +.PHONY: build-docker-testenv clean run-docker-tests test-all-testenv diff --git a/docs/text/how_to_contribute.rst b/docs/text/how_to_contribute.rst index 0633784f..4f169e44 100644 --- a/docs/text/how_to_contribute.rst +++ b/docs/text/how_to_contribute.rst @@ -72,13 +72,24 @@ To test changes across multiple versions of Python, run: .. code:: - tox -r + tox -r -p auto -`tox -r` will execute tests for the Python versions specified in `setup.cfg `_ using the `envlist` variable. For example, if `envlist` is set to `py37, py38`, the test suite will run for Python 3.7 and 3.8 on the local development platform, assuming the binaries for those versions are available locally. The exact Python microversions (e.g. `3.7.1` vs `3.7.2`) depend on what is installed on the local development machine. +This will execute tests for the Python versions specified in `setup.cfg `_ using the `envlist` variable. For example, if `envlist` is set to `py37, py38`, the test suite will run for Python 3.7 and 3.8 on the local development platform, assuming the binaries for those versions are available locally. The exact Python microversions (e.g. `3.7.1` vs `3.7.2`) depend on what is installed on the local development machine. A recommended way to manage multiple Python versions when testing locally is with `pyenv`, which enables organized installation and switching between versions. +In addition to running tests locally, you can also run them in a Dockerized testing environment: + + +.. code:: + + make test-all-testenv + + +This command will initially take some time. However subsequent invokations will be faster, and testing this way ensures a clean, consistent test environment, regardless of your local setup. + + Documentation '''''''''''''