diff --git a/.github/workflows/on_target.yml b/.github/workflows/on_target.yml index 9b9389de..88d879dd 100644 --- a/.github/workflows/on_target.yml +++ b/.github/workflows/on_target.yml @@ -39,7 +39,11 @@ on: description: The run ID of the workflow to fetch artifacts from type: string required: true - run_log_check_tests: + run_functional_tests: + type: boolean + required: true + default: true + run_wifi_location_tests: type: boolean required: true default: true @@ -109,50 +113,33 @@ jobs: --software-version ${{ inputs.artifact_fw_version }} \ hello.nrfcloud.com-${{ inputs.artifact_fw_version }}-thingy91x-nrf91.elf - - name: Run UART tests - if: ${{ inputs.run_log_check_tests }} + - name: Run DUT1 tests + if: ${{ inputs.run_functional_tests }} working-directory: thingy91x-oob/tests/on_target run: | mkdir -p results - pytest -s -v -m "dut1 and uart" \ - --junit-xml=results/test-results-uart.xml \ - tests - env: - SEGGER: ${{ secrets.SEGGER_DUT_1 }} - LOG_FILENAME: oob_uart_test_log - - - name: Run FOTA tests (standard) - if: ${{ inputs.run_fota_tests }} - working-directory: thingy91x-oob/tests/on_target - run: | - pytest -s -v -m "dut1 and fota" \ - --junit-xml=results/test-results-fota.xml \ + pytest -m dut1 \ + --junit-xml=results/test-results-dut1.xml \ + --html=results/test-results-dut1.html --self-contained-html \ tests env: SEGGER: ${{ secrets.SEGGER_DUT_1 }} IMEI: ${{ secrets.IMEI_DUT_1 }} FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }} - LOG_FILENAME: oob_fota_test_log - - - name: Run FOTA tests (FULLMFW) - if: ${{ inputs.run_fullmfwfota_test }} - working-directory: thingy91x-oob/tests/on_target - run: | - pytest -s -v -m "dut1 and fullmfw_fota" \ - --junit-xml=results/test-results-fullmfw-fota.xml \ - tests - env: - SEGGER: ${{ secrets.SEGGER_DUT_1 }} - IMEI: ${{ secrets.IMEI_DUT_1 }} - FINGERPRINT: ${{ secrets.FINGERPRINT_DUT_1 }} - LOG_FILENAME: oob_fullmfw_fota_test_log + LOG_FILENAME: oob_uart_test_log + TEST_FUNCTIONAL: ${{ inputs.run_functional_tests }} + TEST_FOTA: ${{ inputs.run_fota_tests }} + TEST_FOTA_FULLMFW: ${{ inputs.run_fullmfwfota_test }} + TEST_WIFI_LOC: ${{ inputs.run_wifi_location_tests }} + TEST_REPORT_NAME: OOB Firwmare Test Report - name: Run DFU tests if: ${{ inputs.run_dfu_tests }} working-directory: thingy91x-oob/tests/on_target run: | - pytest -s -v -m "dut2 and dfu" \ + pytest -m "dut2 and dfu" \ --junit-xml=results/test-results-dfu.xml \ + --html=results/test-results-dfu.html --self-contained-html \ tests env: SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }} @@ -171,8 +158,9 @@ jobs: if: ${{ inputs.run_connectivity_bridge_tests }} working-directory: thingy91x-oob/tests/on_target run: | - pytest -s -v -m "dut2 and conn_bridge" \ + pytest -m "dut2 and conn_bridge" \ --junit-xml=results/test-results-connectivity-bridge.xml \ + --html=results/test-results-conn-bridge.html --self-contained-html \ tests env: SEGGER_NRF53: ${{ secrets.SEGGER_DUT_2_EXT_DBG }} @@ -182,16 +170,6 @@ jobs: NRF91_CUSTOM_APP_ZIP: artifacts/usb_uart_bridge_app.zip LOG_FILENAME: oob_conn_bridge_test_log - - name: Run Wi-Fi location tests - working-directory: thingy91x-oob/tests/on_target - run: | - mkdir -p results - pytest -s -v -m "dut1 and wifi" \ - --junit-xml=results/test-results-wifi-location.xml \ - tests - env: - SEGGER: ${{ secrets.SEGGER_DUT_1 }} - - name: Results if: always() uses: pmeier/pytest-results-action@v0.7.1 @@ -200,6 +178,15 @@ jobs: summary: true fail-on-empty: true title: OOB FW Test Results + + - name: Create Report Artifact + if: always() + uses: actions/upload-artifact@v4 + id: artifact-report + with: + name: test-report + path: | + thingy91x-oob/tests/on_target/results/*.html - name: Push log files to artifacts if: always() diff --git a/tests/on_target/requirements.txt b/tests/on_target/requirements.txt index 23a58ef0..bb8b5654 100644 --- a/tests/on_target/requirements.txt +++ b/tests/on_target/requirements.txt @@ -1,4 +1,5 @@ pytest +pytest-html pyserial termcolor pyusb diff --git a/tests/on_target/tests/conftest.py b/tests/on_target/tests/conftest.py index fd848038..75ee2ec2 100644 --- a/tests/on_target/tests/conftest.py +++ b/tests/on_target/tests/conftest.py @@ -5,12 +5,14 @@ import os import re -import pytest +import sys import types +import pytest +from pytest_metadata.plugin import metadata_key from utils.flash_tools import recover_device from utils.uart import Uart from utils.hellonrfcloud_fota import HelloNrfCloudFOTA -import sys + sys.path.append(os.getcwd()) from utils.logger import get_logger @@ -23,6 +25,14 @@ FOTADEVICE_IMEI = os.getenv('IMEI') FOTADEVICE_FINGERPRINT = os.getenv('FINGERPRINT') +def pytest_html_report_title(report): + report.title = os.getenv("TEST_REPORT_NAME", "OOB Test Report") + + +def pytest_configure(config): + config.stash[metadata_key]["Board"] = "thingy91x" + config.stash[metadata_key]["Board revision"] = "0.7.0" + def get_uarts(): base_path = "/dev/serial/by-id" try: @@ -62,7 +72,7 @@ def t91x_board(): uart = Uart(log_uart_string, timeout=UART_TIMEOUT) fota = HelloNrfCloudFOTA(device_id=f"oob-{FOTADEVICE_IMEI}", \ - fingerprint=FOTADEVICE_FINGERPRINT) + fingerprint=FOTADEVICE_FINGERPRINT) yield types.SimpleNamespace( uart=uart, diff --git a/tests/on_target/tests/test_fota.py b/tests/on_target/tests/test_fota.py index 7577e390..5bd792d2 100644 --- a/tests/on_target/tests/test_fota.py +++ b/tests/on_target/tests/test_fota.py @@ -76,7 +76,7 @@ def _run_fota(bundleId, fota_type, fotatimeout=APP_FOTA_TIMEOUT, test_fota_resum @pytest.mark.dut1 -@pytest.mark.fota +@pytest.mark.skipif(not os.getenv("TEST_FOTA"), reason="Deselected") def test_app_fota(t91x_board, hex_file, run_fota_fixture): # Get latest APP fota bundle results = t91x_board.fota.get_fota_bundles() @@ -95,7 +95,7 @@ def test_app_fota(t91x_board, hex_file, run_fota_fixture): @pytest.mark.dut1 -@pytest.mark.fota +@pytest.mark.skipif(not os.getenv("TEST_FOTA"), reason="Deselected") def test_delta_mfw_fota(t91x_board, hex_file, run_fota_fixture): # Flash with mfw201 flash_device(os.path.abspath(MFW_201_FILEPATH)) @@ -108,6 +108,6 @@ def test_delta_mfw_fota(t91x_board, hex_file, run_fota_fixture): @pytest.mark.dut1 -@pytest.mark.fullmfw_fota +@pytest.mark.skipif(not os.getenv("TEST_FOTA_FULLMFW"), reason="Deselected") def test_full_mfw_fota(t91x_board, hex_file, run_fota_fixture): run_fota_fixture(FULL_MFW_BUNDLEID, "full", FULL_MFW_FOTA_TIMEOUT) diff --git a/tests/on_target/tests/test_location.py b/tests/on_target/tests/test_location.py index e2228dea..f6f15667 100644 --- a/tests/on_target/tests/test_location.py +++ b/tests/on_target/tests/test_location.py @@ -3,23 +3,24 @@ # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause ########################################################################################## -import pytest -import time import os import re -from utils.flash_tools import flash_device, reset_device import sys +import pytest +from utils.flash_tools import flash_device, reset_device + sys.path.append(os.getcwd()) from utils.logger import get_logger logger = get_logger() -@pytest.mark.wifi @pytest.mark.dut1 +@pytest.mark.skipif(not os.getenv("TEST_WIFI_LOC"), reason="Deselected") def test_wifi_location(t91x_board, hex_file): run_location(t91x_board, hex_file, location_method="Wi-Fi") -@pytest.mark.gnss +@pytest.mark.dut1 +@pytest.mark.skipif(not os.getenv("TEST_GNSS"), reason="Deselected") def test_gnss_location(t91x_board, hex_file): run_location(t91x_board, hex_file, location_method="GNSS") @@ -27,14 +28,14 @@ def run_location(t91x_board, hex_file, location_method): flash_device(os.path.abspath(hex_file)) t91x_board.uart.xfactoryreset() patterns_cloud_connection = [ - "Network connectivity established", - "Connected to Cloud" + "Network connectivity established", + "Connected to Cloud" ] patterns_location = ["Wi-Fi and cellular methods combined"] if location_method == "Wi-Fi" else [] patterns_location = patterns_location + [ - "location_event_handler: Got location: lat:", - "Location search done"] + "location_event_handler: Got location: lat:", + "Location search done"] # Cloud connection t91x_board.uart.flush() diff --git a/tests/on_target/tests/test_uart_output.py b/tests/on_target/tests/test_uart_output.py index 0a70df0f..79844ed8 100644 --- a/tests/on_target/tests/test_uart_output.py +++ b/tests/on_target/tests/test_uart_output.py @@ -15,28 +15,28 @@ logger = get_logger() @pytest.mark.dut1 -@pytest.mark.uart +@pytest.mark.skipif(not os.getenv("TEST_FUNCTIONAL"), reason="Deselected") def test_uart_output(t91x_board, hex_file): flash_device(os.path.abspath(hex_file)) t91x_board.uart.xfactoryreset() patterns_boot = [ - "Network connectivity established", - "Connected to Cloud", - "trigger: frequent_poll_entry: frequent_poll_entry", - "trigger: trigger_work_fn: Sending data sample trigger", - "environmental_module: sample: temp:", - "transport: state_connected_ready_run: Payload", - "Location search done" + "Network connectivity established", + "Connected to Cloud", + "trigger: frequent_poll_entry: frequent_poll_entry", + "trigger: trigger_work_fn: Sending data sample trigger", + "environmental_module: sample: temp:", + "transport: state_connected_ready_run: Payload", + "Location search done" ] patterns_button_press = [ "trigger: frequent_poll_run: Button 1 pressed in frequent poll state, restarting duration timer", "trigger_poll_work_fn: Sending shadow/fota poll trigger" ] patterns_lte_offline = [ - "network: Network connectivity lost", + "network: Network connectivity lost", ] patterns_lte_normal = [ - "network: Network connectivity established", + "network: Network connectivity established", ] # Boot