tests: Add report, restructure

Add html report
Restructure tests, markers

Signed-off-by: Jorgen Kvalvaag <[email protected]>

.github/workflows/on_target.yml

@@ -39,7 +39,11 @@ on:
         description: The run ID of the workflow to fetch artifacts from
         type: string
         required: true
-      run_log_check_tests:
+      run_functional_tests:
+        type: boolean
+        required: true
+        default: true
+      run_wifi_location_tests:
         type: boolean
         required: true
         default: true
@@ -109,50 +113,33 @@ jobs:
               --software-version ${{ inputs.artifact_fw_version }} \
               hello.nrfcloud.com-${{ inputs.artifact_fw_version }}-thingy91x-nrf91.elf
 
-      - name: Run UART tests
-        if: ${{ inputs.run_log_check_tests }}
+      - name: Run DUT1 tests
+        if: ${{ inputs.run_functional_tests }}
         working-directory: thingy91x-oob/tests/on_target
         run: |
           mkdir -p results
-          pytest -s -v -m "dut1 and uart" \
-            --junit-xml=results/test-results-uart.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          LOG_FILENAME: oob_uart_test_log
-
-      - name: Run FOTA tests (standard)
-        if: ${{ inputs.run_fota_tests }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut1 and fota" \
-            --junit-xml=results/test-results-fota.xml \
+          pytest -m dut1 \
+            --junit-xml=results/test-results-dut1.xml \
+            --html=results/test-results-dut1.html --self-contained-html \
             tests
         env:
           SEGGER: ${{ secrets.SEGGER_DUT_1 }}
           IMEI: ${{ secrets.IMEI_DUT_1 }}
           FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
-          LOG_FILENAME: oob_fota_test_log
-
-      - name: Run FOTA tests (FULLMFW)
-        if: ${{ inputs.run_fullmfwfota_test }}
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          pytest -s -v -m "dut1 and fullmfw_fota" \
-            --junit-xml=results/test-results-fullmfw-fota.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-          IMEI: ${{ secrets.IMEI_DUT_1 }}
-          FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }}
-          LOG_FILENAME: oob_fullmfw_fota_test_log
+          LOG_FILENAME: oob_uart_test_log
+          TEST_FUNCTIONAL: ${{ inputs.run_functional_tests }}
+          TEST_FOTA: ${{ inputs.run_fota_tests }}
+          TEST_FOTA_FULLMFW: ${{ inputs.run_fullmfwfota_test }}
+          TEST_WIFI_LOC: ${{ inputs.run_wifi_location_tests }}
+          TEST_REPORT_NAME: OOB Firwmare Test Report 
 
       - name: Run DFU tests
         if: ${{ inputs.run_dfu_tests }}
         working-directory: thingy91x-oob/tests/on_target
         run: |
-          pytest -s -v -m "dut2 and dfu" \
+          pytest -m "dut2 and dfu" \
             --junit-xml=results/test-results-dfu.xml \
+            --html=results/test-results-dfu.html --self-contained-html \
             tests
         env:
           SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
@@ -171,8 +158,9 @@ jobs:
         if: ${{ inputs.run_connectivity_bridge_tests }}
         working-directory: thingy91x-oob/tests/on_target
         run: |
-          pytest -s -v -m "dut2 and conn_bridge" \
+          pytest -m "dut2 and conn_bridge" \
             --junit-xml=results/test-results-connectivity-bridge.xml \
+            --html=results/test-results-conn-bridge.html --self-contained-html \
             tests
         env:
           SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }}
@@ -182,16 +170,6 @@ jobs:
           NRF91_CUSTOM_APP_ZIP: artifacts/usb_uart_bridge_app.zip
           LOG_FILENAME: oob_conn_bridge_test_log
 
-      - name: Run Wi-Fi location tests
-        working-directory: thingy91x-oob/tests/on_target
-        run: |
-          mkdir -p results
-          pytest -s -v -m "dut1 and wifi" \
-            --junit-xml=results/test-results-wifi-location.xml \
-            tests
-        env:
-          SEGGER: ${{ secrets.SEGGER_DUT_1 }}
-
       - name: Results
         if: always()
         uses: pmeier/[email protected]
@@ -200,6 +178,15 @@ jobs:
           summary: true
           fail-on-empty: true
           title: OOB FW Test Results
+
+      - name: Create Report Artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        id: artifact-report
+        with:
+          name: test-report
+          path: |
+            thingy91x-oob/tests/on_target/results/*.html
 
       - name: Push log files to artifacts
         if: always()

tests/on_target/requirements.txt

@@ -1,4 +1,5 @@
 pytest
+pytest-html
 pyserial
 termcolor
 pyusb

tests/on_target/tests/conftest.py

@@ -5,12 +5,14 @@
 
 import os
 import re
-import pytest
+import sys
 import types
+import pytest
+from pytest_metadata.plugin import metadata_key
 from utils.flash_tools import recover_device
 from utils.uart import Uart
 from utils.hellonrfcloud_fota import HelloNrfCloudFOTA
-import sys
+
 sys.path.append(os.getcwd())
 from utils.logger import get_logger
 
@@ -23,6 +25,14 @@
 FOTADEVICE_IMEI = os.getenv('IMEI')
 FOTADEVICE_FINGERPRINT = os.getenv('FINGERPRINT')
 
+def pytest_html_report_title(report):
+    report.title = os.getenv("TEST_REPORT_NAME", "OOB Test Report")
+
+
+def pytest_configure(config):
+    config.stash[metadata_key]["Board"] = "thingy91x"
+    config.stash[metadata_key]["Board revision"] = "0.7.0"
+
 def get_uarts():
     base_path = "/dev/serial/by-id"
     try:
@@ -62,7 +72,7 @@ def t91x_board():
 
     uart = Uart(log_uart_string, timeout=UART_TIMEOUT)
     fota = HelloNrfCloudFOTA(device_id=f"oob-{FOTADEVICE_IMEI}", \
-                                    fingerprint=FOTADEVICE_FINGERPRINT)
+                             fingerprint=FOTADEVICE_FINGERPRINT)
 
     yield types.SimpleNamespace(
 		uart=uart,

tests/on_target/tests/test_fota.py

@@ -76,7 +76,7 @@ def _run_fota(bundleId, fota_type, fotatimeout=APP_FOTA_TIMEOUT, test_fota_resum
 
 
 @pytest.mark.dut1
-@pytest.mark.fota
+@pytest.mark.skipif(not os.getenv("TEST_FOTA"), reason="Deselected")
 def test_app_fota(t91x_board, hex_file, run_fota_fixture):
     # Get latest APP fota bundle
     results = t91x_board.fota.get_fota_bundles()
@@ -95,7 +95,7 @@ def test_app_fota(t91x_board, hex_file, run_fota_fixture):
 
 
 @pytest.mark.dut1
-@pytest.mark.fota
+@pytest.mark.skipif(not os.getenv("TEST_FOTA"), reason="Deselected")
 def test_delta_mfw_fota(t91x_board, hex_file, run_fota_fixture):
     # Flash with mfw201
     flash_device(os.path.abspath(MFW_201_FILEPATH))
@@ -108,6 +108,6 @@ def test_delta_mfw_fota(t91x_board, hex_file, run_fota_fixture):
 
 
 @pytest.mark.dut1
-@pytest.mark.fullmfw_fota
+@pytest.mark.skipif(not os.getenv("TEST_FOTA_FULLMFW"), reason="Deselected")
 def test_full_mfw_fota(t91x_board, hex_file, run_fota_fixture):
     run_fota_fixture(FULL_MFW_BUNDLEID, "full", FULL_MFW_FOTA_TIMEOUT)

tests/on_target/tests/test_location.py

@@ -3,38 +3,39 @@
 # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 ##########################################################################################
 
-import pytest
-import time
 import os
 import re
-from utils.flash_tools import flash_device, reset_device
 import sys
+import pytest
+from utils.flash_tools import flash_device, reset_device
+
 sys.path.append(os.getcwd())
 from utils.logger import get_logger
 
 logger = get_logger()
 
-@pytest.mark.wifi
 @pytest.mark.dut1
+@pytest.mark.skipif(not os.getenv("TEST_WIFI_LOC"), reason="Deselected")
 def test_wifi_location(t91x_board, hex_file):
     run_location(t91x_board, hex_file, location_method="Wi-Fi")
 
-@pytest.mark.gnss
+@pytest.mark.dut1
+@pytest.mark.skipif(not os.getenv("TEST_GNSS"), reason="Deselected")
 def test_gnss_location(t91x_board, hex_file):
     run_location(t91x_board, hex_file, location_method="GNSS")
 
 def run_location(t91x_board, hex_file, location_method):
     flash_device(os.path.abspath(hex_file))
     t91x_board.uart.xfactoryreset()
     patterns_cloud_connection = [
-            "Network connectivity established",
-            "Connected to Cloud"
+        "Network connectivity established",
+        "Connected to Cloud"
     ]
 
     patterns_location = ["Wi-Fi and cellular methods combined"] if location_method == "Wi-Fi" else []
     patterns_location = patterns_location + [
-            "location_event_handler: Got location: lat:",
-            "Location search done"]
+        "location_event_handler: Got location: lat:",
+        "Location search done"]
 
     # Cloud connection
     t91x_board.uart.flush()

tests/on_target/tests/test_uart_output.py

@@ -15,28 +15,28 @@
 logger = get_logger()
 
 @pytest.mark.dut1
-@pytest.mark.uart
+@pytest.mark.skipif(not os.getenv("TEST_FUNCTIONAL"), reason="Deselected")
 def test_uart_output(t91x_board, hex_file):
     flash_device(os.path.abspath(hex_file))
     t91x_board.uart.xfactoryreset()
     patterns_boot = [
-            "Network connectivity established",
-            "Connected to Cloud",
-            "trigger: frequent_poll_entry: frequent_poll_entry",
-            "trigger: trigger_work_fn: Sending data sample trigger",
-            "environmental_module: sample: temp:",
-            "transport: state_connected_ready_run: Payload",
-            "Location search done"
+        "Network connectivity established",
+        "Connected to Cloud",
+        "trigger: frequent_poll_entry: frequent_poll_entry",
+        "trigger: trigger_work_fn: Sending data sample trigger",
+        "environmental_module: sample: temp:",
+        "transport: state_connected_ready_run: Payload",
+        "Location search done"
     ]
     patterns_button_press = [
         "trigger: frequent_poll_run: Button 1 pressed in frequent poll state, restarting duration timer",
         "trigger_poll_work_fn: Sending shadow/fota poll trigger"
     ]
     patterns_lte_offline = [
-            "network: Network connectivity lost",
+        "network: Network connectivity lost",
     ]
     patterns_lte_normal = [
-            "network: Network connectivity established",
+        "network: Network connectivity established",
     ]
 
     # Boot