diff --git a/src/audio/google/CMakeLists.txt b/src/audio/google/CMakeLists.txt index 3936c8041b67..b6f8d3725cd8 100644 --- a/src/audio/google/CMakeLists.txt +++ b/src/audio/google/CMakeLists.txt @@ -29,6 +29,26 @@ if((NOT CONFIG_LIBRARY) OR CONFIG_LIBRARY_STATIC) target_link_libraries(sof PRIVATE c) endif() endif() + + if(CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING) + target_include_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/include) + add_local_sources(sof + google_ctc_audio_processing.c + ) + if(CONFIG_GOOGLE_CTC_AUDIO_PROCESSING_MOCK) + add_local_sources(sof + google_ctc_audio_processing_mock.c + ) + else() + message(INFO "Link with google_ctc_audio_processing") + target_link_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/lib) + target_link_libraries(sof PRIVATE google_ctc_audio_processing) + target_link_libraries(sof PRIVATE c++) + target_link_libraries(sof PRIVATE c++abi) + target_link_libraries(sof PRIVATE m) + target_link_libraries(sof PRIVATE c) + endif() + endif() return() endif() diff --git a/src/audio/google/Kconfig b/src/audio/google/Kconfig index 027e15a2c229..1e60d3487b5e 100644 --- a/src/audio/google/Kconfig +++ b/src/audio/google/Kconfig @@ -80,4 +80,30 @@ config GOOGLE_RTC_AUDIO_PROCESSING_MOCK Mock Google real-time communication audio processing. It allows for compilation check and basic audio flow checking. +config COMP_GOOGLE_CTC_AUDIO_PROCESSING + bool "Google Crosstalk Cancellation Audio processing" + select COMP_BLOB + select GOOGLE_CTC_AUDIO_PROCESSING_MOCK if COMP_STUBS + default n + help + Select for Google crosstalk cancellation audio processing. It + uses the Google real-time audio processing library to perform + crosstalk cancellation. + +config COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES + depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING + int "Number of frames to process for Google Crosstalk Cancellation Audio processing" + default 64 + help + Sets the number of frames to process in the Google crosstalk + cancellation audio processing. + +config GOOGLE_CTC_AUDIO_PROCESSING_MOCK + bool "Google Crosstalk Cancellation Audio processing mock" + default n + depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING + help + Mock Google crosstalk cancellation audio processing. + It allows for compilation check and basic audio flow checking. + endmenu diff --git a/src/audio/google/google_ctc_audio_processing.c b/src/audio/google/google_ctc_audio_processing.c new file mode 100644 index 000000000000..8d5693f5d012 --- /dev/null +++ b/src/audio/google/google_ctc_audio_processing.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2024 Google LLC. +// +// Author: Eddy Hsu +#include +#include +#include +#include + +#include + +LOG_MODULE_REGISTER(google_ctc_audio_processing, CONFIG_SOF_LOG_LEVEL); + +/* bf0e1bbc-dc6a-45fe-bc90-2554cb137ab4 */ +DECLARE_SOF_RT_UUID("google-ctc-audio-processing", google_ctc_audio_processing_uuid, + 0xbf0e1bbc, 0xdc6a, 0x45fe, 0xbc, 0x90, 0x25, 0x54, 0xcb, + 0x13, 0x7a, 0xb4); + +DECLARE_TR_CTX(google_ctc_audio_processing_tr, SOF_UUID(google_ctc_audio_processing_uuid), + LOG_LEVEL_INFO); + +struct google_ctc_audio_processing_comp_data { + struct comp_buffer *input; + struct comp_buffer *output; + uint32_t num_frames; + uint32_t num_channels; + GoogleCtcAudioProcessingState *state; + struct comp_data_blob_handler *tuning_handler; + bool reconfigure; +}; + +static int google_ctc_audio_processing_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct google_ctc_audio_processing_comp_data *cd; + int ret; + + comp_info(dev, "google_ctc_audio_processing_init()"); + + /* Create private component data */ + cd = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*cd)); + if (!cd) { + ret = -ENOMEM; + goto fail; + } + + md->private = cd; + + cd->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES; + + cd->input = + rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->input[0])); + if (!cd->input) { + ret = -ENOMEM; + goto fail; + } + bzero(cd->input, cd->num_frames * sizeof(cd->input[0])); + + cd->output = + rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->output[0])); + if (!cd->output) { + ret = -ENOMEM; + goto fail; + } + bzero(cd->output, cd->num_frames * sizeof(cd->output[0])); + + cd->state = GoogleCtcAudioProcessingCreateWithConfig(cd->num_frames, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_PARTITION_SIZE, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_IMPULSE_SIZE, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_CHUNK_FRAMES, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_SAMPLE_RATE, + /*config=*/NULL, /*config_size=*/0); + + comp_dbg(dev, "google_ctc_audio_processing_init(): Ready"); + + return 0; +fail: + comp_err(dev, "google_ctc_audio_processing_init(): Failed"); + if (cd) { + rfree(cd->input); + rfree(cd->output); + if (cd->state) + GoogleCtcAudioProcessingFree(cd->state); + rfree(cd); + } + rfree(dev); + return ret; +} + +static int google_ctc_audio_processing_free(struct processing_module *mod) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + + GoogleCtcAudioProcessingFree(cd->state); + cd->state = NULL; + rfree(cd->input); + rfree(cd->output); + rfree(cd); + rfree(dev); + return 0; +} + +static int google_ctc_audio_processing_reconfigure(struct processing_module *mod) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + uint8_t *config; + size_t size; + int ret; + + comp_dbg(dev, "google_ctc_audio_processing_reconfigure()"); + + if (!comp_is_current_data_blob_valid(cd->tuning_handler) && + !comp_is_new_data_blob_available(cd->tuning_handler)) { + /* + * The data blob hasn't been available once so far. + * + * This looks redundant since the same check will be done in + * comp_get_data_blob() below. But without this early return, + * hundreds of warn message lines are produced per second by + * comp_get_data_blob() calls until the data blob is arrived. + */ + return 0; + } + + config = comp_get_data_blob(cd->tuning_handler, &size, NULL); + if (size == 0) { + /* No data to be handled */ + return 0; + } + + if (!config) { + comp_err(dev, "google_ctc_audio_processing_reconfigure(): Tuning config not set"); + return -EINVAL; + } + + comp_info(dev, "google_ctc_audio_processing_reconfigure(): New tuning config %p (%zu bytes)", + config, size); + + cd->reconfigure = false; + + uint8_t *google_ctc_audio_processing_config; + size_t google_ctc_audio_processing_config_size; + int num_channels; + bool google_ctc_audio_processing_config_present; + bool num_channels_present; + + GoogleCtcAudioProcessingParseSofConfigMessage(config, size, + &google_ctc_audio_processing_config, + &google_ctc_audio_processing_config_size, + &num_channels, + &google_ctc_audio_processing_config_present, + &num_channels_present); + + if (google_ctc_audio_processing_config_present) { + comp_info(dev, + "google_ctc_audio_processing_reconfigure(): Applying config of size %zu bytes", + google_ctc_audio_processing_config_size); + + ret = GoogleCtcAudioProcessingReconfigure(cd->state, + google_ctc_audio_processing_config, + google_ctc_audio_processing_config_size); + if (ret) { + comp_err(dev, "GoogleCtcAudioProcessingReconfigure failed: %d", + ret); + return ret; + } + } + + if (num_channels_present) { + cd->num_channels = num_channels; + comp_info(dev, + "google_ctc_audio_processing_reconfigure(): Applying num channels %d", + cd->num_channels); + } + + return 0; +} + +static int google_ctc_audio_processing_prepare(struct processing_module *mod) +{ + struct comp_dev *dev = mod->dev; + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + + comp_info(dev, "google_ctc_audio_processing_prepare()"); + + return 0; +} + +static int google_ctc_audio_processing_reset(struct processing_module *mod) +{ + comp_dbg(mod->dev, "google_ctc_audio_processing_reset()"); + + return 0; +} + +static int google_ctc_audio_processing_process(struct processing_module *mod, + struct input_stream_buffer *input_buffers, + int num_input_buffers, + struct output_stream_buffer *output_buffers, + int num_output_buffers) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + int16_t *src; + int16_t *dst; + + struct input_stream_buffer *in_streamb; + struct output_stream_buffer *out_streamb; + struct audio_stream *in_stream, *out_stream; + + comp_dbg(dev, "google_ctc_audio_processing_process()"); + + if (cd->reconfigure) { + ret = google_ctc_audio_processing_reconfigure(mod); + if (ret) + return ret; + } + + in_streamb = &input_buffers[0]; + in_stream = in_streamb->data; + src = audio_stream_get_rptr(in_stream); + out_streamb = &output_buffers[0]; + out_stream = out_streamb->data; + dst = audio_stream_get_rptr(out_stream); + + for (frame = 0; frame < cd->num_frames; frame++) { + for (channel = 0; channel < cd->num_channels; channel++) { + GoogleCtcAudioProcessingProcess(cd->state, src, dst); + + src += num_channels; + dst += num_channels; + } + } + + return 0; +} + +#if CONFIG_IPC_MAJOR_3 +static int google_ctc_audio_processing_cmd_set_data(struct processing_module *mod, + struct sof_ipc_ctrl_data *cdata) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + int ret; + + switch (cdata->cmd) { + case SOF_CTRL_CMD_BINARY: + ret = comp_data_blob_set_cmd(cd->tuning_handler, cdata); + if (ret) + return ret; + /* Accept the new blob immediately so that userspace can write + * the control in quick succession without error. + * This ensures the last successful control write from userspace + * before prepare/copy is applied. + * The config blob is not referenced after reconfigure() returns + * so it is safe to call comp_get_data_blob here which frees the + * old blob. This assumes cmd() and prepare()/copy() cannot run + * concurrently which is the case when there is no preemption. + */ + if (comp_is_new_data_blob_available(cd->tuning_handler)) { + comp_get_data_blob(cd->tuning_handler, NULL, NULL); + cd->reconfigure = true; + } + return 0; + default: + comp_err(mod->dev, + "google_ctc_audio_processing_ctrl_set_data(): Only binary controls supported %d", + cdata->cmd); + return -EINVAL; + } +} + +static int google_ctc_audio_processing_cmd_get_data(struct processing_module *mod, + struct sof_ipc_ctrl_data *cdata, + size_t max_data_size) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + + comp_info(mod->dev, "google_ctc_audio_processing_ctrl_get_data(): %u", cdata->cmd); + + switch (cdata->cmd) { + case SOF_CTRL_CMD_BINARY: + return comp_data_blob_get_cmd(cd->tuning_handler, cdata, max_data_size); + default: + comp_err(mod->dev, + "google_ctc_audio_processing_ctrl_get_data(): Only binary controls supported %d", + cdata->cmd); + return -EINVAL; + } +} +#endif + +static int google_ctc_audio_processing_set_config(struct processing_module *mod, uint32_t param_id, + enum module_cfg_fragment_position pos, + uint32_t data_offset_size, + const uint8_t *fragment, + size_t fragment_size, uint8_t *response, + size_t response_size) +{ +#if CONFIG_IPC_MAJOR_4 + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + int ret; + + switch (param_id) { + case SOF_IPC4_SWITCH_CONTROL_PARAM_ID: + case SOF_IPC4_ENUM_CONTROL_PARAM_ID: + comp_err(mod->dev, "google_ctc_audio_processing_set_data(): Only binary controls supported"); + return -EINVAL; + } + + ret = comp_data_blob_set(cd->tuning_handler, pos, data_offset_size, + fragment, fragment_size); + if (ret) + return ret; + + /* Accept the new blob immediately so that userspace can write + * the control in quick succession without error. + * This ensures the last successful control write from userspace + * before prepare/copy is applied. + * The config blob is not referenced after reconfigure() returns + * so it is safe to call comp_get_data_blob here which frees the + * old blob. This assumes cmd() and prepare()/copy() cannot run + * concurrently which is the case when there is no preemption. + * + * Note from review: A race condition is possible and should be + * further investigated and fixed. + */ + if (comp_is_new_data_blob_available(cd->tuning_handler)) { + comp_get_data_blob(cd->tuning_handler, NULL, NULL); + cd->reconfigure = true; + } + + return 0; +#elif CONFIG_IPC_MAJOR_3 + struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; + + return google_ctc_audio_processing_cmd_set_data(mod, cdata); +#endif +} + +static int google_ctc_audio_processing_get_config(struct processing_module *mod, + uint32_t param_id, uint32_t *data_offset_size, + uint8_t *fragment, size_t fragment_size) +{ +#if CONFIG_IPC_MAJOR_4 + comp_err(mod->dev, "google_ctc_audio_processing_get_config(): Not supported"); + return -EINVAL; +#elif CONFIG_IPC_MAJOR_3 + struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; + + return google_ctc_audio_processing_cmd_get_data(mod, cdata, fragment_size); +#endif +} + +static struct module_interface google_ctc_audio_processing_interface = { + .init = google_ctc_audio_processing_init, + .free = google_ctc_audio_processing_free, + .process_audio_stream = google_ctc_audio_processing_process, + .prepare = google_ctc_audio_processing_prepare, + .set_configuration = google_ctc_audio_processing_set_config, + .get_configuration = google_ctc_audio_processing_get_config, + .reset = google_ctc_audio_processing_reset, +}; + +DECLARE_MODULE_ADAPTER(google_ctc_audio_processing_interface, + google_ctc_audio_processing_uuid, google_ctc_audio_processing_tr); +SOF_MODULE_INIT(google_ctc_audio_processing, + sys_comp_module_google_ctc_audio_processing_interface_init); diff --git a/src/audio/google/google_ctc_audio_processing_mock.c b/src/audio/google/google_ctc_audio_processing_mock.c new file mode 100644 index 000000000000..2c3f55b06096 --- /dev/null +++ b/src/audio/google/google_ctc_audio_processing_mock.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2024 Google LLC. +// +// Author: Eddy Hsu +#include "google_ctc_audio_processing.h" + +#include +#include +#include +#include + +#include "ipc/topology.h" + +struct GoogleCtcAudioProcessingState { + int num_frames; + int partition_size; + int impulse_size; + int chunk_frames; + int sample_rate; + int is_symmetric; +}; + +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void) +{ + struct GoogleCtcAudioProcessingState *s = + rballoc(0, SOF_MEM_CAPS_RAM, sizeof(GoogleCtcAudioProcessingState)); + if (!s) + return NULL; + + s->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES; + s->partition_size = 64; + s->impulse_size = 256; + s->chunk_frames = 480; + s->sample_rate = 48000; + s->is_symmetric = 0; + return s; +} + +void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state) +{ + if (state) + rfree(state); +} + +int GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *const state, + const int16_t *const src, + int16_t *const dest) +{ + memcpy_s(dest, sizeof(int16_t) * state->num_frames, + src, sizeof(int16_t) * state->num_frames); + return 0; +} diff --git a/third_party/include/google_ctc_audio_processing.h b/third_party/include/google_ctc_audio_processing.h new file mode 100644 index 000000000000..b18d0488bfb9 --- /dev/null +++ b/third_party/include/google_ctc_audio_processing.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright(c) 2024 Google LLC. + * + * Author: Eddy Hsu + */ +#ifndef GOOGLE_CTC_AUDIO_PROCESSING_H +#define GOOGLE_CTC_AUDIO_PROCESSING_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct GoogleCtcAudioProcessingState GoogleCtcAudioProcessingState; + +// Creates an instance of GoogleCtcAudioProcessing with the tuning embedded in +// the library. If creation fails, NULL is returned. +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void); + +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreateWithConfig( + int num_frames, int partition_size, int impulse_size, int chunk_frames, + int sample_rate, int is_symmetric, const uint8_t *const config, + int config_size); + +// Frees all allocated resources in `state`. +void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state); + +// Apply CTC on `src` and produce result in `dest`. +void GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *const state, + const float *const src, float *const dest); + + +// Reconfigure the audio processing. +// Returns 0 if success and non zero if failure. +int GoogleCtcAudioProcessingReconfigure( + GoogleCtcAudioProcessingState *const state, const uint8_t *const config, + int config_size); + +#ifdef __cplusplus +} +#endif + +#endif // GOOGLE_CTC_AUDIO_PROCESSING_H