Enable coverage collecting and reporting (#250)

ansible · Aug 14, 2024 · a615966 · a615966
1 parent 5a3f586
commit a615966
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 32 deletions.
diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml
@@ -38,13 +38,14 @@ jobs:
           default_python: "3.9"
           other_names: |
             lint
-            py39:tox -e py39-sanity;tox -e py39-unit; tox -e py39-integration
-            py310:tox -e py310-sanity;tox -e py310-unit; tox -e py310-integration
-            py311:tox -e py311-sanity;tox -e py311-unit; tox -e py311-integration
-            py312:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration
-            py39-macos:tox -e py39-sanity;tox -e py39-unit; tox -e py39-integration
-            py312-macos:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration
-            py311-linux-arm64:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration
+            py39:tox -e py39-sanity;tox -e py39-unit;tox -e py39-integration;tox -e coverage
+            py312:tox -e py312-sanity;tox -e py312-unit;tox -e py312-integration;tox -e coverage
+          # py310:tox -e py310-sanity;tox -e py310-unit; tox -e py310-integration; tox -e coverage
+          # py311:tox -e py311-sanity;tox -e py311-unit; tox -e py311-integration; tox -e coverage
+          # py312:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration; tox -e coverage
+          # py39-macos:tox -e py39-sanity;tox -e py39-unit; tox -e py39-integration; tox -e coverage
+          # py312-macos:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration; tox -e coverage
+          # py311-linux-arm64:tox -e py312-sanity;tox -e py312-unit; tox -e py312-integration; tox -e coverage
           # ^ arm64 runner is using py311 for matching python version used in AAP 2.5
           platforms: linux,macos,linux-arm64:ubuntu-24.04-arm64-2core
           skip_explode: "1"
@@ -123,15 +124,18 @@ jobs:
       - run: ${{ matrix.command3 }}
         if: ${{ matrix.command3 }}
 
+      - run: ${{ matrix.command4 }}
+        if: ${{ matrix.command4 }}
+
       - name: Archive logs
         uses: actions/upload-artifact@v4
         with:
           name: logs-${{ matrix.name }}.zip
           if-no-files-found: error
           path: |
             ansible_collections/ansible/eda/.tox/**/log/
-            ansible_collections/ansible/eda/.tox/**/.coverage*
             ansible_collections/ansible/eda/.tox/**/coverage.xml
+            ansible_collections/ansible/eda/tests/output/reports/coverage.xml
 
       - name: Report failure if git reports dirty status
         run: |
@@ -178,25 +182,27 @@ jobs:
           name: logs.zip
           path: .
 
-      # - name: Check for expected number of coverage.xml reports
-      #   run: |
-      #     JOBS_PRODUCING_COVERAGE=8
-      #     if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then
-      #       echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)"
-      #       exit 1
-      #     fi
-
-      # - name: Upload coverage data
-      #   uses: codecov/codecov-action@v4
-      #   with:
-      #     name: ${{ matrix.name }}
-      #     # verbose: true # optional (default = false)
-      #     fail_ci_if_error: true
-      #     use_oidc: true # cspell:ignore oidc
-
-      # - name: Check codecov.io status
-      #   if: github.event_name == 'pull_request'
-      #   uses: coactions/codecov-status@main
+      - name: Check for expected number of coverage reports
+        run: |
+          JOBS_PRODUCING_COVERAGE=2
+          FIND_FILES="find */tests/output/reports -name 'coverage.xml'"
+          if [ "$($FIND_FILES | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then
+            echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $($FIND_FILES | xargs echo)"
+            exit 0
+          fi
+
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v4
+        with:
+          name: ${{ matrix.name }}
+          # verbose: true # optional (default = false)
+          fail_ci_if_error: true
+          use_oidc: true # cspell:ignore oidc
+          files: "*/tests/output/reports/coverage.xml"
+
+      - name: Check codecov.io status
+        if: github.event_name == 'pull_request'
+        uses: coactions/codecov-status@main
 
       - name: Decide whether the needed jobs succeeded or failed
         uses: re-actors/alls-green@release/v1

diff --git a/.gitignore b/.gitignore
@@ -20,3 +20,7 @@ tests/integration/event_source_kafka/*.jks
 
 # Builds
 *.tar.gz
+
+# Coverage
+.coverage*
+*coverage.combined
diff --git a/codecov.yml b/codecov.yml
@@ -0,0 +1,8 @@
+---
+comment: false
+coverage:
+  status:
+    patch: true
+    project:
+      default:
+        threshold: 0.5%
diff --git a/extensions/eda/plugins/event_source/__init__.py b/extensions/eda/plugins/event_source/__init__.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,30 @@
 [tool.black]
 target-version = ["py310"]
 
+# Keep this default because xml/report do not know to use load it from config file:
+# data_file = ".coverage"
+# [tool.coverage.paths]
+# source = ["plugins", "extensions"]
+
+# [tool.coverage.report]
+# exclude_also = ["pragma: no cover", "if TYPE_CHECKING:"]
+# omit = ["test/*"]
+# # Increase it just so it would pass on any single-python run
+# fail_under = 92
+# skip_covered = true
+# skip_empty = true
+# # During development we might remove code (files) with coverage data, and we dont want to fail:
+# ignore_errors = true
+# show_missing = true
+
+# [tool.coverage.run]
+# source = ["src"]
+# # Do not use branch until bug is fixes:
+# # https://github.com/nedbat/coveragepy/issues/605
+# # branch = true
+# parallel = true
+# concurrency = ["multiprocessing", "thread"]
+
 [tool.isort]
 profile = "black"
 multi_line_output = 3

diff --git a/test_requirements.txt b/test_requirements.txt
@@ -1,4 +1,7 @@
 -r requirements.txt
+coverage-enable-subprocess>=1.0 # see https://github.com/nedbat/coveragepy/issues/1341#issuecomment-1228942657
+coverage==6.5.0; python_version < "3.10"  # ansible-core will complain
+coverage[toml] == 7.3.2; python_version >= "3.10" # ansible-core will complain
 pytest>=7.0.0
 asyncmock>=0.4.2
 pytest-asyncio>=0.20.0

diff --git a/tox.ini b/tox.ini
@@ -1,10 +1,10 @@
 # Recommended usage of this file is detailed in https://github.com/ansible/eda-partner-testing/blob/main/README.md.
 # The linter paths can be changed, but may result in false passes.
 # {posargs} in this case would be the path to collection root relative from the .github/workflows dir (`../..`)
-# cspell: ignore TOXPYTHON setenv passenv REQPASS PYTHONPYCACHEPREFIX PYTHONIOENCODING PYTHONBREAKPOINT notest
+# cspell: ignore TOXPYTHON setenv passenv REQPASS PYTHONPYCACHEPREFIX PYTHONIOENCODING PYTHONBREAKPOINT notest envdir envname toxworkdir
 
 [tox]
-envlist = lint, darglint, unit, sanity, integration
+envlist = lint, darglint, unit, sanity, integration, coverage
 skipsdist = true # this repo is not a python package
 isolated_build = true
 requires =
@@ -23,10 +23,13 @@ commands =
   tox -e sanity,unit,integration {posargs}
 allowlist_externals =
   bash
+  sh
 isolated_build = true
 ;   # See https://stackoverflow.com/a/76884052/99834
 ;   py310: true
 setenv =
+    ; COVERAGE_FILE = {env:COVERAGE_FILE:{envdir}/.coverage.{envname}}
+    ; COVERAGE_PROCESS_START={toxinidir}/pyproject.toml
     PIP_NO_BUILD_ISOLATION = 1
     PYTHONUNBUFFERED = 1
 passenv =
@@ -71,7 +74,11 @@ description =
 commands =
   # risky: not safe for development it affects user setup
   ansible-galaxy collection install .
-  pytest tests/integration -vvv -s {posargs}
+  # use same coverage location as ansible-test:
+  coverage run --data-file=tests/output/coverage/integration.coverage -m pytest tests/integration -vvv -s {posargs} -k test_url_check_source_sanity
+  ; sh -c "coverage combine -a -q --data-file=tests/output/coverage/integration.combined tests/output/coverage/integration*.coverage"
+  ; sh -c "coverage report --data-file=tests/output/coverage/integration.combined --show-missing --skip-empty --include=extensions/*,plugins/*,tests/\*"
+  ; bash -c "coverage xml --data-file=tests/output/coverage/integration.coverage -o {envdir}/coverage.xml --fail-under=0"
 
 [testenv:unit,py{39,310,311,312,313}-unit]
 description =
@@ -80,7 +87,9 @@ description =
 commands =
   # fail-fast if psycopg in not properly installed.
   python3 -c "import psycopg"
-  ansible-test units --venv -v --num-workers 1
+  sh -c "ansible-test units --requirements --target-python default -v --num-workers 1 --coverage --truncate 0 {posargs}"
+  ansible-test coverage report --requirements --omit '.tox/*,tests/*' --color --all --show-missing -v --truncate 0
+  ansible-test coverage combine --export={envdir}
 
 [testenv:sanity,py{39,310,311,312,313}-sanity]
 deps = -r test_requirements.txt
@@ -89,4 +98,14 @@ description =
   py{39,310,311,312,313}: with {basepython}
 commands =
   ansible --version
-  ansible-test sanity {posargs}
+  ansible-test sanity -v --requirements --coverage --truncate 0 {posargs}
+
+[testenv:coverage]
+description = Produce final coverage report for GHA
+deps = -r test_requirements.txt
+commands =
+  ansible-test coverage report --requirements --omit '.tox/*,tests/*' --color --all --show-missing -v --truncate 0
+  # produces: tests/output/coverage/coverage.combined
+  ansible-test coverage combine --export={envdir}
+  # produces tests/output/reports/coverage.xml
+  ansible-test coverage xml --requirements