From a61596623942e67cbc807aa654db4c40d60636e8 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Wed, 14 Aug 2024 09:38:51 +0100 Subject: [PATCH] Enable coverage collecting and reporting (#250) --- .github/workflows/tox.yml | 60 ++++++++++--------- .gitignore | 4 ++ codecov.yml | 8 +++ .../eda/plugins/event_source/__init__.py | 0 pyproject.toml | 24 ++++++++ test_requirements.txt | 3 + tox.ini | 29 +++++++-- 7 files changed, 96 insertions(+), 32 deletions(-) create mode 100644 codecov.yml create mode 100644 extensions/eda/plugins/event_source/__init__.py diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 0474505c..a066b205 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -38,13 +38,14 @@ jobs: default_python: "3.9" other_names: | lint - py39:tox -e py39-sanity;tox -e py39-unit; tox -e py39-integration - py310:tox -e py310-sanity;tox -e py310-unit; tox -e py310-integration - py311:tox -e py311-sanity;tox -e py311-unit; tox -e py311-integration - py312:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration - py39-macos:tox -e py39-sanity;tox -e py39-unit; tox -e py39-integration - py312-macos:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration - py311-linux-arm64:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration + py39:tox -e py39-sanity;tox -e py39-unit;tox -e py39-integration;tox -e coverage + py312:tox -e py312-sanity;tox -e py312-unit;tox -e py312-integration;tox -e coverage + # py310:tox -e py310-sanity;tox -e py310-unit; tox -e py310-integration; tox -e coverage + # py311:tox -e py311-sanity;tox -e py311-unit; tox -e py311-integration; tox -e coverage + # py312:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration; tox -e coverage + # py39-macos:tox -e py39-sanity;tox -e py39-unit; tox -e py39-integration; tox -e coverage + # py312-macos:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration; tox -e coverage + # py311-linux-arm64:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration; tox -e coverage # ^ arm64 runner is using py311 for matching python version used in AAP 2.5 platforms: linux,macos,linux-arm64:ubuntu-24.04-arm64-2core skip_explode: "1" @@ -123,6 +124,9 @@ jobs: - run: ${{ matrix.command3 }} if: ${{ matrix.command3 }} + - run: ${{ matrix.command4 }} + if: ${{ matrix.command4 }} + - name: Archive logs uses: actions/upload-artifact@v4 with: @@ -130,8 +134,8 @@ jobs: if-no-files-found: error path: | ansible_collections/ansible/eda/.tox/**/log/ - ansible_collections/ansible/eda/.tox/**/.coverage* ansible_collections/ansible/eda/.tox/**/coverage.xml + ansible_collections/ansible/eda/tests/output/reports/coverage.xml - name: Report failure if git reports dirty status run: | @@ -178,25 +182,27 @@ jobs: name: logs.zip path: . - # - name: Check for expected number of coverage.xml reports - # run: | - # JOBS_PRODUCING_COVERAGE=8 - # if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then - # echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)" - # exit 1 - # fi - - # - name: Upload coverage data - # uses: codecov/codecov-action@v4 - # with: - # name: ${{ matrix.name }} - # # verbose: true # optional (default = false) - # fail_ci_if_error: true - # use_oidc: true # cspell:ignore oidc - - # - name: Check codecov.io status - # if: github.event_name == 'pull_request' - # uses: coactions/codecov-status@main + - name: Check for expected number of coverage reports + run: | + JOBS_PRODUCING_COVERAGE=2 + FIND_FILES="find */tests/output/reports -name 'coverage.xml'" + if [ "$($FIND_FILES | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then + echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $($FIND_FILES | xargs echo)" + exit 0 + fi + + - name: Upload coverage data + uses: codecov/codecov-action@v4 + with: + name: ${{ matrix.name }} + # verbose: true # optional (default = false) + fail_ci_if_error: true + use_oidc: true # cspell:ignore oidc + files: "*/tests/output/reports/coverage.xml" + + - name: Check codecov.io status + if: github.event_name == 'pull_request' + uses: coactions/codecov-status@main - name: Decide whether the needed jobs succeeded or failed uses: re-actors/alls-green@release/v1 diff --git a/.gitignore b/.gitignore index a79ec97b..c92c5ef4 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,7 @@ tests/integration/event_source_kafka/*.jks # Builds *.tar.gz + +# Coverage +.coverage* +*coverage.combined diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..ccb3a37e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +--- +comment: false +coverage: + status: + patch: true + project: + default: + threshold: 0.5% diff --git a/extensions/eda/plugins/event_source/__init__.py b/extensions/eda/plugins/event_source/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml index aaf52dae..b5d00fa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,30 @@ [tool.black] target-version = ["py310"] +# Keep this default because xml/report do not know to use load it from config file: +# data_file = ".coverage" +# [tool.coverage.paths] +# source = ["plugins", "extensions"] + +# [tool.coverage.report] +# exclude_also = ["pragma: no cover", "if TYPE_CHECKING:"] +# omit = ["test/*"] +# # Increase it just so it would pass on any single-python run +# fail_under = 92 +# skip_covered = true +# skip_empty = true +# # During development we might remove code (files) with coverage data, and we dont want to fail: +# ignore_errors = true +# show_missing = true + +# [tool.coverage.run] +# source = ["src"] +# # Do not use branch until bug is fixes: +# # https://github.com/nedbat/coveragepy/issues/605 +# # branch = true +# parallel = true +# concurrency = ["multiprocessing", "thread"] + [tool.isort] profile = "black" multi_line_output = 3 diff --git a/test_requirements.txt b/test_requirements.txt index cf3c7bdd..c66df362 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,7 @@ -r requirements.txt +coverage-enable-subprocess>=1.0 # see https://github.com/nedbat/coveragepy/issues/1341#issuecomment-1228942657 +coverage==6.5.0; python_version < "3.10" # ansible-core will complain +coverage[toml] == 7.3.2; python_version >= "3.10" # ansible-core will complain pytest>=7.0.0 asyncmock>=0.4.2 pytest-asyncio>=0.20.0 diff --git a/tox.ini b/tox.ini index eeeb43f8..7b2c9480 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,10 @@ # Recommended usage of this file is detailed in https://github.com/ansible/eda-partner-testing/blob/main/README.md. # The linter paths can be changed, but may result in false passes. # {posargs} in this case would be the path to collection root relative from the .github/workflows dir (`../..`) -# cspell: ignore TOXPYTHON setenv passenv REQPASS PYTHONPYCACHEPREFIX PYTHONIOENCODING PYTHONBREAKPOINT notest +# cspell: ignore TOXPYTHON setenv passenv REQPASS PYTHONPYCACHEPREFIX PYTHONIOENCODING PYTHONBREAKPOINT notest envdir envname toxworkdir [tox] -envlist = lint, darglint, unit, sanity, integration +envlist = lint, darglint, unit, sanity, integration, coverage skipsdist = true # this repo is not a python package isolated_build = true requires = @@ -23,10 +23,13 @@ commands = tox -e sanity,unit,integration {posargs} allowlist_externals = bash + sh isolated_build = true ; # See https://stackoverflow.com/a/76884052/99834 ; py310: true setenv = + ; COVERAGE_FILE = {env:COVERAGE_FILE:{envdir}/.coverage.{envname}} + ; COVERAGE_PROCESS_START={toxinidir}/pyproject.toml PIP_NO_BUILD_ISOLATION = 1 PYTHONUNBUFFERED = 1 passenv = @@ -71,7 +74,11 @@ description = commands = # risky: not safe for development it affects user setup ansible-galaxy collection install . - pytest tests/integration -vvv -s {posargs} + # use same coverage location as ansible-test: + coverage run --data-file=tests/output/coverage/integration.coverage -m pytest tests/integration -vvv -s {posargs} -k test_url_check_source_sanity + ; sh -c "coverage combine -a -q --data-file=tests/output/coverage/integration.combined tests/output/coverage/integration*.coverage" + ; sh -c "coverage report --data-file=tests/output/coverage/integration.combined --show-missing --skip-empty --include=extensions/*,plugins/*,tests/\*" + ; bash -c "coverage xml --data-file=tests/output/coverage/integration.coverage -o {envdir}/coverage.xml --fail-under=0" [testenv:unit,py{39,310,311,312,313}-unit] description = @@ -80,7 +87,9 @@ description = commands = # fail-fast if psycopg in not properly installed. python3 -c "import psycopg" - ansible-test units --venv -v --num-workers 1 + sh -c "ansible-test units --requirements --target-python default -v --num-workers 1 --coverage --truncate 0 {posargs}" + ansible-test coverage report --requirements --omit '.tox/*,tests/*' --color --all --show-missing -v --truncate 0 + ansible-test coverage combine --export={envdir} [testenv:sanity,py{39,310,311,312,313}-sanity] deps = -r test_requirements.txt @@ -89,4 +98,14 @@ description = py{39,310,311,312,313}: with {basepython} commands = ansible --version - ansible-test sanity {posargs} + ansible-test sanity -v --requirements --coverage --truncate 0 {posargs} + +[testenv:coverage] +description = Produce final coverage report for GHA +deps = -r test_requirements.txt +commands = + ansible-test coverage report --requirements --omit '.tox/*,tests/*' --color --all --show-missing -v --truncate 0 + # produces: tests/output/coverage/coverage.combined + ansible-test coverage combine --export={envdir} + # produces tests/output/reports/coverage.xml + ansible-test coverage xml --requirements