app: Enable modem traces in release builds

Enable modem traces in release builds:
 - Only send modem traces to memfault in debug build
 - Traces are default on for all builds, need to look into the energy
   penalty of doing this.
 - Added docs on how to dump modem traces.

Signed-off-by: Simen S. Røstad <[email protected]>

.github/workflows/build.yml

@@ -153,7 +153,7 @@ jobs:
           echo CONFIG_MEMFAULT_NCS_FW_VERSION=\"${{ env.VERSION }}+debug\" >> overlay-memfault-debug.conf
           echo CONFIG_MEMFAULT_NCS_FW_TYPE=\"${{ env.MEMFAULT_SW_TYPE }}\" >> overlay-memfault-debug.conf
           echo CONFIG_APP_MEMFAULT_UPLOAD_METRICS_ON_CLOUD_READY=y >> overlay-memfault-debug.conf
-          west build -p -b thingy91x/nrf9151/ns -p --sysbuild -- -DEXTRA_CONF_FILE="overlay-memfault-debug.conf;overlay-modemtrace.conf;overlay-etb.conf"
+          west build -p -b thingy91x/nrf9151/ns -p --sysbuild -- -DEXTRA_CONF_FILE="overlay-memfault-debug.conf;overlay-modemtrace-to-memfault.conf;overlay-etb.conf"
 
       - name: Rename debug artifacts
         if: ${{ inputs.build_debug }}

README.md

@@ -81,3 +81,24 @@ west flash --erase
 | Solid          | Configured | Device has received a LED configuration      | NA                                                  |
 | Blinking rapid | Red        | Fatal error, the device will reboot          | NA                                                  |
 | Blinking slow  | Red        | Irrecoverable Fatal error                    | NA                                                  |
+
+### Modem Traces
+
+Modem traces are enabled by default on the Thingy:91 device. These traces can be output to UART for analysis using the **nRF Connect for Desktop Cellular Monitor** application.
+
+#### Steps to Capture and Dump Modem Traces:
+
+1. **Connect to a Serial Terminal**
+   - Connect your Thingy:91 device to a serial terminal on **UART 0**. This will allow you to interact with the device's shell commands. You might need to push **Button 1** to wake the UART up.
+
+2. **Set Up Cellular Monitor Application**
+   - Open the [Cellular Monitor Application](https://docs.nordicsemi.com/bundle/nrf-connect-cellularmonitor/page/index.html).
+   - Connect the Thingy:91 to the application, select **UART 1** as the trace output, and click **Start Traces** to begin capturing modem activity.
+
+3. **Dump Traces via UART**
+   - Use the following shell commands in the connected serial terminal to manage and dump the modem traces on **UART 1**:
+
+   ```shell
+   modem_trace stop         # Stop modem tracing if running
+   modem_trace size         # Check the size of stored traces
+   modem_trace dump_uart    # Dump traces to UART 1 for analysis

app/overlay-modemtrace-to-memfault.conf

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2024 Nordic Semiconductor
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+#
+
+CONFIG_APP_MEMFAULT_INCLUDE_MODEM_TRACES=y
+
+# Disable traces by default, let the Memfault module enable and set the trace the log level.
+CONFIG_NRF_MODEM_LIB_TRACE_LEVEL_OFF=y
+
+# Set nRF Cloud CoAP security tag to a dev tag to enable decoded DTLS traces.
+# It is required that the nRF Cloud CoAP server certificate has been provisioned to the sec tag.
+CONFIG_NRF_CLOUD_COAP_SEC_TAG=2147483667
+CONFIG_NRF_CLOUD_COAP_JWT_SEC_TAG=4242

app/prj.conf

@@ -71,6 +71,7 @@ CONFIG_NRF_MODEM_LIB=y
 # AT commands interface
 CONFIG_UART_INTERRUPT_DRIVEN=y
 CONFIG_SHELL=y
+CONFIG_SHELL_WILDCARD=n
 CONFIG_SHELL_STACK_SIZE=1560
 CONFIG_AT_SHELL=y
 
@@ -247,3 +248,19 @@ CONFIG_TASK_WDT_CHANNELS=9
 
 # Device power management
 CONFIG_PM_DEVICE=y
+
+# Modem tracing
+CONFIG_FCB=y
+CONFIG_FLASH_MAP=y
+CONFIG_NRF_MODEM_LIB_TRACE=y
+CONFIG_NRF_MODEM_LIB_SHELL_TRACE=y
+CONFIG_NRF_MODEM_LIB_TRACE_LEVEL_OVERRIDE=y
+# Start IP traces after boot with LTE and IP trace level
+CONFIG_NRF_MODEM_LIB_TRACE_LEVEL_LTE_AND_IP=y
+CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH=y
+CONFIG_NRF_MODEM_TRACE_FLASH_NOSPACE_ERASE_OLDEST=y
+CONFIG_NRF_MODEM_LIB_TRACE_STACK_SIZE=1024
+# Maximum number of sectors that the trace backend can handle
+CONFIG_NRF_MODEM_LIB_TRACE_FLASH_SECTORS=255
+# Modem trace flash partition size with 255 sectors of 4KB each
+CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH_PARTITION_SIZE=0xFF000

app/sample.yaml

@@ -18,7 +18,7 @@ tests:
       - thingy91x/nrf9151/ns
     extra_configs:
       - CONFIG_MEMFAULT_NCS_PROJECT_KEY="PROJECTKEY"
-    extra_args: EXTRA_CONF_FILE="overlay-memfault.conf;overlay-modemtrace.conf;overlay-etb.conf"
+    extra_args: EXTRA_CONF_FILE="overlay-memfault.conf;overlay-modemtrace-to-memfault.conf;overlay-etb.conf"
     tags: ci_build
   app.build.bootloader_update:
     build_only: true

app/src/modules/memfault/Kconfig.memfault

@@ -17,6 +17,12 @@ menuconfig APP_MEMFAULT
 
 if APP_MEMFAULT
 
+config APP_MEMFAULT_INCLUDE_MODEM_TRACES
+	bool "Modem traces to memfault"
+	select MEMFAULT_CDR_ENABLE
+	help
+	  Include modem traces with Memfault coredumps.
+
 config APP_MEMFAULT_THREAD_STACK_SIZE
 	int "Thread stack size"
 	default 1536

app/src/modules/memfault/memfault.c

@@ -37,7 +37,7 @@ static void task_wdt_callback(int channel_id, void *user_data)
 	SEND_FATAL_ERROR_WATCHDOG_TIMEOUT();
 }
 
-#if defined(CONFIG_NRF_MODEM_LIB_TRACE) && defined(CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH)
+#if defined(CONFIG_APP_MEMFAULT_INCLUDE_MODEM_TRACES)
 
 static const char *mimetypes[] = { MEMFAULT_CDR_BINARY };
 
@@ -175,7 +175,7 @@ static void on_connected(void)
 		return;
 	}
 
-#if defined(CONFIG_NRF_MODEM_LIB_TRACE) && defined(CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH)
+#if defined(CONFIG_APP_MEMFAULT_INCLUDE_MODEM_TRACES)
 	/* If there was a coredump, also send modem trace */
 
 	if (has_coredump) {
@@ -186,7 +186,7 @@ static void on_connected(void)
 
 	memfault_zephyr_port_post_data();
 
-#if defined(CONFIG_NRF_MODEM_LIB_TRACE) && defined(CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH)
+#if defined(CONFIG_APP_MEMFAULT_INCLUDE_MODEM_TRACES)
 	int err	= modem_trace_enable();
 
 	if (err) {