Implement SonarCloud integrations (#3001)

Co-authored-by: Jan Klopper <[email protected]>

.github/scripts/coverage_file_fixer.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+
+# This script fixes the filename attribute in a coverage.xml file for SonarCloud coverage analysis
+# These filenames should be relative to the base dir and not the coverage source directory
+
+
+import argparse
+import xml.etree.ElementTree as etree
+from pathlib import Path
+
+
+def path_prefixer(file: Path, prefix: Path) -> None:
+    xml = file.read_text()
+    root = etree.fromstring(xml)  # noqa: S314
+
+    for element in root.findall(".//*[@filename]"):
+        filename = element.get("filename")
+        if filename is not None:
+            element.set("filename", prefix.joinpath(filename).as_posix())
+
+    file.write_text(etree.tostring(root).decode())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Prefix paths in coverage.xml files")
+    parser.add_argument("file", type=Path, help="Path to the coverage.xml file.")
+    parser.add_argument("prefix", type=Path, help="Path to prefix the filenames with.")
+    args = parser.parse_args()
+
+    path_prefixer(Path(args.file), Path(args.prefix))
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/boefjes-tests.yml

@@ -0,0 +1,39 @@
+name: Boefjes Test (with coverage)
+
+on:
+  workflow_call:
+
+jobs:
+  Tests:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip" # caching pip dependencies
+
+      - name: Install pip
+        run: python3 -m pip install --upgrade pip wheel
+
+      - name: Install dev requirements
+        run: grep -v git+https:// requirements-dev.txt | pip install -r /dev/stdin && grep git+https:// requirements-dev.txt | pip install -r /dev/stdin
+        working-directory: boefjes/
+
+      - name: Install plugin requirements
+        run: find boefjes/plugins/ -name requirements.txt -execdir pip install -r requirements.txt \;
+        working-directory: boefjes/
+
+      - name: Install Octopoes
+        run: cd octopoes && python setup.py bdist_wheel && pip install dist/octopoes*.whl
+
+      - name: Run pytests
+        run: python3 -m pytest --cov boefjes/ --cov-report xml --cov-branch boefjes/tests
+
+      - name: Upload coverage as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: boefjes-coverage-unit
+          path: coverage.xml

.github/workflows/bytes-tests.yml

@@ -0,0 +1,20 @@
+name: Bytes Tests (with coverage)
+
+on:
+  workflow_call:
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run unit tests
+        run: make utest
+        working-directory: bytes/
+
+      - name: Upload coverage as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bytes-coverage-unit
+          path: bytes/coverage.xml

.github/workflows/mula-tests.yml

@@ -0,0 +1,20 @@
+name: Mula Tests (with coverage)
+
+on:
+  workflow_call:
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run unit tests
+        run: make utest
+        working-directory: mula/
+
+      - name: Upload coverage as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: mula-coverage-unit
+          path: mula/coverage.xml

.github/workflows/octopoes-tests.yml

@@ -0,0 +1,29 @@
+name: Octopoes Tests (with coverage)
+
+on:
+  workflow_call:
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip" # caching pip dependencies
+
+      - name: Install requirements
+        run: pip install -r requirements-dev.txt
+        working-directory: octopoes/
+
+      - name: Run unit tests
+        run: pytest --cov octopoes/ octopoes/tests/
+
+      - name: Upload coverage as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: octopoes-coverage-unit
+          path: coverage.xml

.github/workflows/pre_commit_checks.yml

@@ -1,6 +1,7 @@
 name: Pre-commit checks
 
 on:
+  workflow_call:
   push:
     branches:
       - "main"
@@ -23,7 +24,7 @@ jobs:
           cache: pip
 
       - name: Install pre-commit
-        run: pip install pre-commit==3.6.0
+        run: pip install pre-commit==3.8.0
 
       - uses: actions/cache@v4
         with:

.github/workflows/rocky-tests.yml

@@ -0,0 +1,23 @@
+name: Rocky Tests (with coverage)
+
+on:
+  workflow_call:
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build image
+        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/.ci/docker-compose.yml build --build-arg USER_UID=1001 --build-arg USER_GID=1001 --build-arg PYTHON_VERSION=3.10 rocky_tests
+
+      - name: Run tests
+        run: docker compose -f rocky/.ci/docker-compose.yml run --rm rocky_tests
+
+      - name: Upload coverage as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: rocky-coverage-unit
+          path: rocky/coverage.xml

.github/workflows/sonar-cloud.yml

@@ -0,0 +1,94 @@
+name: SonarCloud
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - "main"
+  pull_request:
+
+jobs:
+  octopoes-tests:
+    uses: minvws/nl-kat-coordination/.github/workflows/octopoes-tests.yml@feature/sonar-cloud
+  bytes-tests:
+    uses: minvws/nl-kat-coordination/.github/workflows/bytes-tests.yml@feature/sonar-cloud
+  mula-tests:
+    uses: minvws/nl-kat-coordination/.github/workflows/mula-tests.yml@feature/sonar-cloud
+  rocky-tests:
+    uses: minvws/nl-kat-coordination/.github/workflows/rocky-tests.yml@feature/sonar-cloud
+  boefjes-tests:
+    uses: minvws/nl-kat-coordination/.github/workflows/boefjes-tests.yml@feature/sonar-cloud
+
+  fix-coverage-reports:
+    runs-on: ubuntu-latest
+
+    needs:
+      - octopoes-tests
+      - mula-tests
+      - bytes-tests
+      - rocky-tests
+      - boefjes-tests
+
+    strategy:
+      matrix:
+        module:
+          - name: octopoes
+            prefix_path: "."
+          - name: mula
+            prefix_path: "mula/scheduler/"
+          - name: bytes
+            prefix_path: "bytes/"
+          - name: rocky
+            prefix_path: "rocky/"
+          - name: boefjes
+            prefix_path: "boefjes/"
+
+    steps:
+      - name: Checkout coverage file fix script
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: .github/scripts/coverage_file_fixer.py
+
+      - name: Download coverage file
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.module['name'] }}-coverage-unit
+          path: ${{ matrix.module['name'] }}-coverage-unit
+
+      - name: Fix coverage report sources
+        uses: Mudlet/xmlstarlet-action@master
+        with:
+          args: edit --inplace --update "coverage/sources" --value "/github/workspace/${{ matrix.module['name'] }}/" "${{ matrix.module['name'] }}-coverage-unit/coverage.xml"
+
+      - name: Fix coverage file
+        run: python "${{ github.workspace }}/.github/scripts/coverage_file_fixer.py" "${{ matrix.module['name'] }}-coverage-unit/coverage.xml" "${{ matrix.module['prefix_path'] }}"
+
+      - name: Upload fixed coverage file
+        uses: actions/upload-artifact@v4
+        with:
+          name: "${{ matrix.module['name'] }}-coverage-unit-fixed"
+          path: "${{ matrix.module['name'] }}-coverage-unit/coverage.xml"
+
+  sonar-cloud:
+    runs-on: ubuntu-latest
+
+    needs:
+      - fix-coverage-reports
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "*-coverage-unit-fixed"
+
+      - name: SonarCloud
+        uses: SonarSource/[email protected]
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}