Merge branch 'develop' into feature/dfu

examples/audio_mux/src/app_conf.h

@@ -50,7 +50,7 @@
 #endif
 
 #ifndef appconfUSB_ENABLED
-#define appconfUSB_ENABLED         (appconfUSB_INPUT || appconfI2S_OUTPUT)
+#define appconfUSB_ENABLED         (appconfUSB_INPUT || appconfUSB_OUTPUT)
 #endif
 
 #ifndef appconfUSB_AUDIO_ENABLED

examples/ffd/ext/ffd_usb_audio_testing.cmake

@@ -12,6 +12,7 @@ set(BYPASS_AUDIOPIPELINE_DEFINITIONS
     appconfAUDIO_PIPELINE_SKIP_IC_AND_VNR=1
     appconfAUDIO_PIPELINE_SKIP_NS=1
     appconfAUDIO_PIPELINE_SKIP_AGC=1
+    appconfLOWPOWER_ENABLED=0
 )
 
 #**********************

examples/ffd/inference/wanson/wanson_inf_eng_port.c

@@ -96,42 +96,42 @@ void wanson_engine_proc_keyword_result(const char **text, int id)
         }
     }
 
-#if appconfSSD1306_DISPLAY_ENABLED
-    // some temporary fixes to the strings returned
-    switch (id) {
-        case 50:
-            // Clear the display
-            ssd1306_display_ascii_to_bitmap("\0");
-            break;
-        case 3:
-            // fix capital "On"
-            ssd1306_display_ascii_to_bitmap("Switch on the TV\0");
-            break;
-        case 15:
-            // fix lower case "speed"
-            // fix word wrapping
-            ssd1306_display_ascii_to_bitmap("Speed up the   fan\0");
-            break;
-        case 16:
-            // fix lower case "slow"
-            ssd1306_display_ascii_to_bitmap("Slow down the fan\0");
-            break;
-        case 17:
-            // fix lower case "set"
-            // fix word wrapping
-            ssd1306_display_ascii_to_bitmap("Set higher    temperature\0");
-            break;
-        case 18:
-            // fix lower case "set"
-            // fix word wrapping
-            ssd1306_display_ascii_to_bitmap("Set lower     temperature\0");
-            break;
-        default:
-            if(text != NULL) {
-                ssd1306_display_ascii_to_bitmap((char *)*text);
-            }
-    }
-#endif
+// #if appconfSSD1306_DISPLAY_ENABLED
+//     // some temporary fixes to the strings returned
+//     switch (id) {
+//         case 50:
+//             // Clear the display
+//             ssd1306_display_ascii_to_bitmap("\0");
+//             break;
+//         case 3:
+//             // fix capital "On"
+//             ssd1306_display_ascii_to_bitmap("Switch on the TV\0");
+//             break;
+//         case 15:
+//             // fix lower case "speed"
+//             // fix word wrapping
+//             ssd1306_display_ascii_to_bitmap("Speed up the   fan\0");
+//             break;
+//         case 16:
+//             // fix lower case "slow"
+//             ssd1306_display_ascii_to_bitmap("Slow down the fan\0");
+//             break;
+//         case 17:
+//             // fix lower case "set"
+//             // fix word wrapping
+//             ssd1306_display_ascii_to_bitmap("Set higher    temperature\0");
+//             break;
+//         case 18:
+//             // fix lower case "set"
+//             // fix word wrapping
+//             ssd1306_display_ascii_to_bitmap("Set lower     temperature\0");
+//             break;
+//         default:
+//             if(text != NULL) {
+//                 ssd1306_display_ascii_to_bitmap((char *)*text);
+//             }
+//     }
+// #endif
 }
 
 int32_t inference_engine_create(uint32_t priority, void *args)

examples/ffd/src/app_conf.h

@@ -11,6 +11,7 @@
 #define appconfINTENT_MODEL_RUNNER_SAMPLES_PORT   3
 #define appconfI2C_MASTER_RPC_PORT                4
 
+#define appconfPOWER_STATE_PORT                  12
 #define appconfWANSON_READY_SYNC_PORT             16
 
 /* Application tile specifiers */
@@ -57,6 +58,11 @@
 #define appconfINTENT_QUEUE_LEN     10
 #endif
 
+/* Maximum number of detected intents to hold */
+#ifndef appconfWAKEUP_TRIGGER_LEN
+#define appconfWAKEUP_TRIGGER_LEN     1
+#endif
+
 /* External wakeup pin edge on intent found.  0 for rising edge, 1 for falling edge */
 #ifndef appconfINTENT_WAKEUP_EDGE_TYPE
 #define appconfINTENT_WAKEUP_EDGE_TYPE     0
@@ -95,6 +101,15 @@
 #define appconfAUDIO_PIPELINE_SKIP_IC_AND_VNR   0
 #endif
 
+#ifndef appconfLOWPOWER_ENABLED
+#define appconfLOWPOWER_ENABLED   1
+#endif
+
+#define appconfPOWER_VNR_THRESHOLD           (0.5)
+#define appconfPOWER_LOW_ENERGY_THRESHOLD    (0.2)
+#define appconfPOWER_HIGH_ENERGY_THRESHOLD   (4.0)
+#define appconfPOWER_FULL_HOLD_DURATION      (3000)   // milliseconds
+
 #ifndef appconfAUDIO_PIPELINE_SKIP_NS
 #define appconfAUDIO_PIPELINE_SKIP_NS   0
 #endif
@@ -160,6 +175,7 @@
 #define appconfQSPI_FLASH_TASK_PRIORITY             (configMAX_PRIORITIES - 1)
 #define appconfSSD1306_TASK_PRIORITY                (configMAX_PRIORITIES / 2 - 1)
 #define appconfLED_HEARTBEAT_TASK_PRIORITY          (configMAX_PRIORITIES / 2 - 1)
+#define appconfPOWER_STATUS_PRIORITY    	        (configMAX_PRIORITIES / 2 - 1)
 
 #include "app_conf_check.h"
 

examples/ffd/src/audio_pipeline/audio_pipeline.c

@@ -23,22 +23,25 @@
 /* App headers */
 #include "app_conf.h"
 #include "audio_pipeline/audio_pipeline.h"
+#include "power/power_state.h"
+
+#define VNR_AGC_THRESHOLD (0.5)
+#define EMA_ENERGY_ALPHA  (0.25)
 
 /* Note: Changing the order here will effect the channel order for
  * audio_pipeline_input() and audio_pipeline_output()
  */
 typedef struct {
     int32_t samples[appconfAUDIO_PIPELINE_CHANNELS][appconfAUDIO_PIPELINE_FRAME_ADVANCE];
     int32_t mic_samples_passthrough[appconfAUDIO_PIPELINE_CHANNELS][appconfAUDIO_PIPELINE_FRAME_ADVANCE];
-    int32_t vnr_pred_flag;
+    float_s32_t vnr_pred;
+    float_s32_t ema_energy;
 } frame_data_t;
 
 #if appconfAUDIO_PIPELINE_FRAME_ADVANCE != 240
 #error This pipeline is only configured for 240 frame advance
 #endif
 
-#define VNR_AGC_THRESHOLD (0.5)
-
 typedef struct ic_stage_ctx {
     ic_state_t state;
 } ic_stage_ctx_t;
@@ -61,6 +64,10 @@ static vnr_pred_stage_ctx_t DWORD_ALIGNED vnr_pred_stage_state = {};
 static ns_stage_ctx_t DWORD_ALIGNED ns_stage_state = {};
 static agc_stage_ctx_t DWORD_ALIGNED agc_stage_state = {};
 
+static uq2_30 ema_energy_alpha_q30 = Q30(EMA_ENERGY_ALPHA); 
+
+static power_data_t* power_app_data = 0; 
+
 static void *audio_pipeline_input_i(void *input_app_data)
 {
     frame_data_t *frame_data;
@@ -71,7 +78,8 @@ static void *audio_pipeline_input_i(void *input_app_data)
                        (int32_t **)frame_data->samples,
                        2,
                        appconfAUDIO_PIPELINE_FRAME_ADVANCE);
-    frame_data->vnr_pred_flag = 0;
+    frame_data->vnr_pred = f32_to_float_s32(0.0);
+    frame_data->ema_energy = f32_to_float_s32(0.0);
 
     memcpy(frame_data->mic_samples_passthrough, frame_data->samples, sizeof(frame_data->mic_samples_passthrough));
 
@@ -81,7 +89,13 @@ static void *audio_pipeline_input_i(void *input_app_data)
 static int audio_pipeline_output_i(frame_data_t *frame_data,
                                    void *output_app_data)
 {
-    return audio_pipeline_output(output_app_data,
+    power_app_data->vnr_pred =  float_s32_to_float(frame_data->vnr_pred);
+    power_app_data->ema_energy =  float_s32_to_float(frame_data->ema_energy);
+
+    //NOTE: passing power_app_data to callback instead of output_app_data.
+    //       They should be the same pointer.
+    assert(power_app_data == output_app_data);
+    return audio_pipeline_output(power_app_data,
                                (int32_t **)frame_data->samples,
                                4,
                                appconfAUDIO_PIPELINE_FRAME_ADVANCE);
@@ -115,13 +129,7 @@ static void stage_vnr_and_ic(frame_data_t *frame_data)
     vnr_inference(&ie_output, &feature_patch);
     vnr_pred_state->output_vnr_pred = float_s32_ema(vnr_pred_state->output_vnr_pred, ie_output, vnr_pred_state->pred_alpha_q30);
 
-#if 0
-    rtos_printf("VNR OUTPUT PRED: %ld %d\n", vnr_pred_stage_state.vnr_pred_state.output_vnr_pred.mant, vnr_pred_stage_state.vnr_pred_state.output_vnr_pred.exp);
-    rtos_printf("VNR INPUT PRED: %ld %d\n", vnr_pred_stage_state.vnr_pred_state.input_vnr_pred.mant, vnr_pred_stage_state.vnr_pred_state.input_vnr_pred.exp);
-#endif
-
-    float_s32_t agc_vnr_threshold = f32_to_float_s32(VNR_AGC_THRESHOLD);
-    frame_data->vnr_pred_flag = float_s32_gt(vnr_pred_stage_state.vnr_pred_state.output_vnr_pred, agc_vnr_threshold);
+    frame_data->vnr_pred = vnr_pred_stage_state.vnr_pred_state.output_vnr_pred;
 
     ic_adapt(&ic_stage_state.state, vnr_pred_stage_state.vnr_pred_state.input_vnr_pred);
 
@@ -142,6 +150,16 @@ static void stage_ns(frame_data_t *frame_data)
                 frame_data->samples[0]);
     memcpy(frame_data->samples, ns_output, appconfAUDIO_PIPELINE_FRAME_ADVANCE * sizeof(int32_t));
 #endif
+
+#if appconfLOWPOWER_ENABLED
+    // Compute exponential moving average of frame energy
+    bfp_s32_t B;
+    bfp_s32_init(&B, &frame_data->samples[0][0], -31, appconfAUDIO_PIPELINE_FRAME_ADVANCE, 1);
+    float_s32_t energy = float_s64_to_float_s32(bfp_s32_energy(&B));
+    frame_data->ema_energy = float_s32_ema(frame_data->ema_energy, energy, ema_energy_alpha_q30);
+#else
+   frame_data->ema_energy = f32_to_float_s32(0.0);;
+#endif
 }
 
 static void stage_agc(frame_data_t *frame_data)
@@ -152,7 +170,9 @@ static void stage_agc(frame_data_t *frame_data)
     int32_t DWORD_ALIGNED agc_output[appconfAUDIO_PIPELINE_FRAME_ADVANCE];
     configASSERT(AGC_FRAME_ADVANCE == appconfAUDIO_PIPELINE_FRAME_ADVANCE);
 
-    agc_stage_state.md.vnr_flag = frame_data->vnr_pred_flag;
+    agc_stage_state.md.vnr_flag = float_s32_gt(frame_data->vnr_pred, f32_to_float_s32(VNR_AGC_THRESHOLD));
+
+
 
     agc_process_frame(
             &agc_stage_state.state,
@@ -201,6 +221,7 @@ void audio_pipeline_init(
 
     initialize_pipeline_stages();
 
+    power_app_data = (power_data_t *) output_app_data;
     generic_pipeline_init((pipeline_input_t)audio_pipeline_input_i,
                         (pipeline_output_t)audio_pipeline_output_i,
                         input_app_data,

examples/ffd/src/main.c

@@ -29,10 +29,14 @@
 #include "xcore_device_memory.h"
 #include "ssd1306_rtos_support.h"
 #include "intent_handler/intent_handler.h"
+#include "power/power_state.h"
+#include "power/power_status.h"
 
 extern void startup_task(void *arg);
 extern void tile_common_init(chanend_t c);
 
+static power_data_t wakeup_app_data = {};
+
 //void uart_write(char data) {} //API for Wanson's Debug
 
 __attribute__((weak))
@@ -71,10 +75,14 @@ int audio_pipeline_output(void *output_app_data,
                           size_t ch_count,
                           size_t frame_count)
 {
-    (void) output_app_data;
-
+    power_state_t power_state = POWER_STATE_FULL;
+#if appconfLOWPOWER_ENABLED
+    power_state = power_state_data_add((power_data_t *)output_app_data);
+#endif
 #if appconfINFERENCE_ENABLED
-    inference_engine_sample_push((int32_t *)output_audio_frames, frame_count);
+    if (power_state == POWER_STATE_FULL) {
+        inference_engine_sample_push((int32_t *)output_audio_frames, frame_count);
+    }
 #endif
 
     return AUDIO_PIPELINE_FREE_FRAME;
@@ -87,6 +95,14 @@ void vApplicationMallocFailedHook(void)
     for(;;);
 }
 
+// static void mem_analysis(void)
+// {
+// 	for (;;) {
+// 		rtos_printf("Tile[%d]:\n\tMinimum heap free: %d\n\tCurrent heap free: %d\n", THIS_XCORE_TILE, xPortGetMinimumEverFreeHeapSize(), xPortGetFreeHeapSize());
+// 		vTaskDelay(pdMS_TO_TICKS(5000));
+// 	}
+// }
+
 __attribute__((weak))
 void startup_task(void *arg)
 {
@@ -121,13 +137,20 @@ void startup_task(void *arg)
         rtos_intertile_rx_data(intertile_ctx, &ret, sizeof(ret));
     }
 #endif
-    audio_pipeline_init(NULL, NULL);
+    audio_pipeline_init(NULL, &wakeup_app_data);
+#if appconfLOWPOWER_ENABLED
+    power_state_init();
+#endif
 #endif
 
 #if ON_TILE(0)
     led_heartbeat_create(appconfLED_HEARTBEAT_TASK_PRIORITY, NULL);
+#if appconfLOWPOWER_ENABLED
+    power_status_create(appconfPOWER_STATUS_PRIORITY, NULL);
+#endif
 #endif
 
+    //mem_analysis();
     vTaskSuspend(NULL);
     while(1){;} /* Trap */
 }

examples/ffd/src/power/power_state.c

@@ -0,0 +1,53 @@
+// Copyright (c) 2022 XMOS LIMITED. This Software is subject to the terms of the
+// XMOS Public License: Version 1
+
+/* STD headers */
+#include <platform.h>
+#include <xs1.h>
+
+/* FreeRTOS headers */
+#include "FreeRTOS.h"
+#include "task.h"
+
+/* App headers */
+#include "app_conf.h"
+#include "platform/driver_instances.h"
+#include "power/power_state.h"
+
+static TimerHandle_t power_state_timer;
+static power_state_t power_state = POWER_STATE_FULL;
+
+void vWakeupStateCallback(TimerHandle_t pxTimer)
+{
+    power_state_set(POWER_STATE_LOW);
+    xTimerStop(power_state_timer, 0);
+}
+
+void power_state_set(power_state_t state) {
+    if (state != power_state) {
+        power_state = state;
+        rtos_intertile_tx(intertile_ctx, appconfPOWER_STATE_PORT, &power_state, sizeof(power_state));
+    }
+}
+
+void power_state_init() {
+    power_state_timer = xTimerCreate(
+        "disp_reset",
+        pdMS_TO_TICKS(appconfPOWER_FULL_HOLD_DURATION),
+        pdFALSE,
+        NULL,
+        vWakeupStateCallback);
+
+    power_state_set(POWER_STATE_LOW);
+}
+
+power_state_t power_state_data_add(power_data_t *data) {
+    if ((data->ema_energy >= appconfPOWER_HIGH_ENERGY_THRESHOLD) ||
+        ((data->ema_energy >= appconfPOWER_LOW_ENERGY_THRESHOLD) &&
+         (data->vnr_pred >= appconfPOWER_VNR_THRESHOLD))) {
+            power_state_set(POWER_STATE_FULL);
+            xTimerReset(power_state_timer, 0);
+    }
+
+    return power_state;
+}