[WIP] comp: Add initial Google CTC component

Introduce a new component to perform crosstalk-cancellation on the capture path. Signed-off-by: Eddy Hsu <[email protected]>
thesofproject · Jan 3, 2024 · 6cb6a6a · 6cb6a6a
1 parent 9315ada
commit 6cb6a6a
Show file tree

Hide file tree

Showing 6 changed files with 450 additions and 0 deletions.
diff --git a/src/audio/google/CMakeLists.txt b/src/audio/google/CMakeLists.txt
@@ -29,6 +29,26 @@ if((NOT CONFIG_LIBRARY) OR CONFIG_LIBRARY_STATIC)
 		target_link_libraries(sof PRIVATE c)
 	  endif()
 	endif()
+
+	if(CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING)
+	  target_include_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/include)
+	  add_local_sources(sof
+		google_ctc_audio_processing.c
+		)
+	  if(CONFIG_GOOGLE_CTC_AUDIO_PROCESSING_MOCK)
+		add_local_sources(sof
+		  google_ctc_audio_processing_mock.c
+		  )
+	  else()
+		message(INFO "Link with google_ctc_audio_processing")
+		target_link_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/lib)
+		target_link_libraries(sof PRIVATE google_ctc_audio_processing)
+		target_link_libraries(sof PRIVATE c++)
+		target_link_libraries(sof PRIVATE c++abi)
+		target_link_libraries(sof PRIVATE m)
+		target_link_libraries(sof PRIVATE c)
+	  endif()
+	endif()
 	return()
 endif()
 
diff --git a/src/audio/google/Kconfig b/src/audio/google/Kconfig
@@ -80,4 +80,30 @@ config GOOGLE_RTC_AUDIO_PROCESSING_MOCK
 	  Mock Google real-time communication audio processing.
 	  It allows for compilation check and basic audio flow checking.
 
+config COMP_GOOGLE_CTC_AUDIO_PROCESSING
+	bool "Google Crosstalk Cancellation Audio processing"
+	select COMP_BLOB
+	select GOOGLE_CTC_AUDIO_PROCESSING_MOCK if COMP_STUBS
+	default n
+	help
+	  Select for Google crosstalk cancellation audio processing. It
+	  uses the Google real-time audio processing library to perform
+	  crosstalk cancellation.
+
+config COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES
+	depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING
+	int "Number of frames to process for Google Crosstalk Cancellation Audio processing"
+	default 64
+	help
+	  Sets the number of frames to process in the Google crosstalk
+	  cancellation audio processing.
+
+config GOOGLE_CTC_AUDIO_PROCESSING_MOCK
+	bool "Google Crosstalk Cancellation Audio processing mock"
+	default n
+	depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING
+	help
+	  Mock Google crosstalk cancellation audio processing.
+	  It allows for compilation check and basic audio flow checking.
+
 endmenu
diff --git a/src/audio/google/google_ctc_audio_processing.c b/src/audio/google/google_ctc_audio_processing.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Copyright(c) 2024 Google LLC.
+//
+// Author: Eddy Hsu <[email protected]>
+#include <sof/audio/buffer.h>
+#include <sof/audio/component.h>
+#include <sof/audio/format.h>
+#include <sof/audio/pipeline.h>
+
+#include <google_ctc_audio_processing.h>
+
+struct comp_driver comp_ctc_audio_processing;
+
+/* bf0e1bbc-dc6a-45fe-bc90-2554cb137ab4 */
+DECLARE_SOF_RT_UUID("google-ctc-audio-processing", google_ctc_audio_processing_uuid,
+		    0xbf0e1bbc, 0xdc6a, 0x45fe, 0xbc, 0x90, 0x25, 0x54, 0xcb,
+		    0x13, 0x7a, 0xb4);
+
+DECLARE_TR_CTX(google_ctc_audio_processing_tr, SOF_UUID(google_ctc_audio_processing_uuid),
+	       LOG_LEVEL_INFO);
+
+struct google_ctc_audio_processing_comp_data {
+	struct comp_data_blob_handler *model_handler;
+	struct comp_buffer *input;
+	struct comp_buffer *output;
+	uint32_t num_frames;
+	GoogleCtcAudioProcessingState *state;
+	struct comp_data_blob_handler *tuning_handler;
+	bool reconfigure;
+};
+
+struct comp_driver comp_ctc_audio_processing = {
+	.uid = SOF_RT_UUID(google_ctc_audio_processing_uuid),
+	.tctx = &google_ctc_audio_processing_tr,
+	.ops = {
+		.create = ctc_audio_processing_create,
+		.free = ctc_audio_processing_free,
+		.params = NULL,
+		.cmd = ctc_audio_processing_cmd,
+		.trigger = ctc_audio_processing_trigger,
+		.prepare = ctc_audio_processing_prepare,
+		.reset = ctc_audio_processing_reset,
+		.copy = ctc_audio_processing_copy,
+	},
+};
+
+static SHARED_DATA struct comp_driver_info comp_ctc_audio_processing_info = {
+	.drv = &comp_ctc_audio_processing,
+};
+
+static void sys_comp_ctc_audio_processing_init(void)
+{
+	comp_register(platform_shared_get(&comp_ctc_audio_processing_info,
+					  sizeof(comp_ctc_audio_processing_info)));
+}
+
+static struct comp_dev *ctc_audio_processing_create(const struct comp_driver *drv,
+						    struct comp_ipc_config *config,
+						    void *spec)
+{
+	struct comp_dev *dev;
+	struct ctc_audio_processing_comp_data *cd;
+	struct ipc_config_process *ipc_ctc = spec;
+
+	comp_cl_info(&comp_ctc_audio_processing, "ctc_audio_processing_create()");
+
+	dev = comp_alloc(drv, sizeof(*dev));
+	if (!dev)
+		return NULL;
+	dev->ipc_config = *config;
+
+	cd = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*cd));
+	if (!cd) {
+		rfree(dev);
+		return NULL;
+	}
+
+	cd->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES;
+
+	cd->input =
+		rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->input[0]));
+	if (!cd->input)
+		goto fail;
+	bzero(cd->input, cd->num_frames * sizeof(cd->input[0]));
+
+	cd->output =
+		rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->output[0]));
+	if (!cd->output)
+		goto fail;
+	bzero(cd->output, cd->num_frames * sizeof(cd->output[0]));
+
+	cd->state = GoogleCtcAudioProcessingCreate();
+
+	comp_set_drvdata(dev, cd);
+
+	dev->state = COMP_STATE_READY;
+
+	comp_dbg(dev, "crosstalk cancellation created");
+
+	return dev;
+fail:
+	comp_err(dev, "google_ctc_audio_processing_create(): Failed");
+	if (cd) {
+		rfree(cd->input);
+		rfree(cd->output);
+		if (cd->state)
+			GoogleCtcAudioProcessingFree(cd->state);
+		rfree(cd);
+	}
+	rfree(dev);
+	return NULL;
+}
+
+static void ctc_audio_processing_free(struct comp_dev *dev)
+{
+	struct comp_data *cd = comp_get_drvdata(dev);
+
+	GoogleCtcAudioProcessingFree(cd->state);
+	cd->state = NULL;
+	rfree(cd->input);
+	rfree(cd->output);
+	rfree(cd);
+	rfree(dev);
+}
+
+static int ctc_audio_processing_trigger(struct comp_dev *dev, int cmd)
+{
+	comp_dbg(dev, "crosstalk cancellation got trigger cmd %d", cmd);
+	return comp_set_state(dev, cmd);
+}
+
+static int ctc_audio_processing_prepare(struct comp_dev *dev)
+{
+	int ret;
+
+	comp_info(dev, "google_ctc_audio_processing_prepare()");
+
+	ret = comp_set_state(dev, COMP_TRIGGER_PREPARE);
+	if (ret < 0)
+		return ret;
+
+	if (ret == COMP_STATUS_STATE_ALREADY_SET)
+		return PPL_STATUS_PATH_STOP;
+
+	comp_dbg(dev, "crosstalk cancellation prepared");
+	return 0;
+}
+
+static int ctc_audio_processing_reset(struct comp_dev *dev)
+{
+	return comp_set_state(dev, COMP_TRIGGER_RESET);
+}
+
+static int ctc_audio_processing_copy(struct comp_dev *dev)
+{
+	struct comp_copy_limits cl;
+	struct comp_buffer *source;
+	struct comp_buffer *sink;
+	struct ctc_audio_processing_comp_data *cd = comp_get_drvdata(dev);
+	int frame;
+	int channel;
+	uint32_t buff_frag = 0;
+	int16_t *src;
+	int16_t *dst;
+
+	comp_dbg(dev, "ctc_audio_processing_copy()");
+
+	source = list_first_item(&dev->bsource_list, struct comp_buffer, sink_list);
+	sink = list_first_item(&dev->bsink_list, struct comp_buffer, source_list);
+
+	comp_get_copy_limits_with_lock(source, sink, &cl);
+
+	buffer_invalidate(source, cl.source_bytes);
+
+	for (frame = 0; frame < cl.frames; frame++) {
+		for (channel = 0; channel < sink->stream.channels; channel++) {
+			src = audio_stream_read_frag_s16(&source->stream, buff_frag);
+			dst = audio_stream_write_frag_s16(&sink->stream, buff_frag);
+
+			GoogleCtcAudioProcessingProcess(cd->state, src, dst);
+
+			++buff_frag;
+		}
+	}
+
+	buffer_writeback(sink, cl.sink_bytes);
+
+	comp_update_buffer_produce(sink, cl.sink_bytes);
+	comp_update_buffer_consume(source, cl.source_bytes);
+
+	return 0;
+}
+
+static int ctc_audio_processing_cmd_set_data(struct comp_dev *dev,
+					     struct sof_ipc_ctrl_data *cdata)
+{
+	struct ctc_audio_processing_comp_data *cd = comp_get_drvdata(dev);
+
+	if (cdata->cmd != SOF_CTRL_CMD_BINARY) {
+		comp_err(dev, "ctc_audio_processing_cmd_set_data(): invalid cmd %d", cdata->cmd);
+		return -EINVAL;
+	}
+
+	ret = comp_data_blob_set_cmd(cd->tuning_handler, cdata);
+	if (ret)
+		return ret;
+
+	/* Accept the new blob immediately so that userspace can write
+	 * the control in quick succession without error.
+	 * This ensures the last successful control write from userspace
+	 * before prepare/copy is applied.
+	 * The config blob is not referenced after reconfigure() returns
+	 * so it is safe to call comp_get_data_blob here which frees the
+	 * old blob. This assumes cmd() and prepare()/copy() cannot run
+	 * concurrently which is the case when there is no preemption.
+	 */
+	if (comp_is_new_data_blob_available(cd->tuning_handler)) {
+		comp_get_data_blob(cd->tuning_handler, NULL, NULL);
+		cd->reconfigure = true;
+	}
+
+	comp_dbg(dev, "crosstalk cancellation new settings");
+	return 0;
+}
+
+static int ctc_audio_processing_cmd_get_data(struct comp_dev *dev,
+					     struct sof_ipc_ctrl_data *cdata,
+					     int max_data_size)
+{
+	struct ctc_audio_processing_comp_data *cd = comp_get_drvdata(dev);
+
+	if (cdata->cmd != SOF_CTRL_CMD_BINARY) {
+		comp_err(dev, "ctc_audio_processing_cmd_get_data(): invalid cmd %d", cdata->cmd);
+		return -EINVAL;
+	}
+
+	return comp_data_blob_get_cmd(cd->tuning_handler, cdata, max_data_size);
+}
+
+static int ctc_audio_processing_cmd(struct comp_dev *dev, int cmd, void *data,
+				    int max_data_size)
+{
+	struct sof_ipc_ctrl_data *cdata = data;
+	int ret = 0;
+
+	switch (cmd) {
+	case COMP_CMD_SET_VALUE:
+	case COMP_CMD_GET_VALUE:
+		return 0;
+	case COMP_CMD_SET_DATA:
+		ret = ctc_audio_processing_cmd_set_data(dev, cdata);
+		break;
+	case COMP_CMD_GET_DATA:
+		ret = ctc_audio_processing_cmd_get_data(dev, cdata, max_data_size);
+		break;
+	default:
+		comp_err(dev, "ctc_audio_processing_cmd(): unhandled command %d", cmd);
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static SHARED_DATA struct comp_driver_info google_ctc_audio_processing_info = {
+	.drv = &google_ctc_audio_processing,
+};
+
+UT_STATIC void sys_comp_google_ctc_audio_processing_init(void)
+{
+	comp_register(platform_shared_get(&google_ctc_audio_processing_info,
+					  sizeof(google_ctc_audio_processing_info)));
+}
+
+DECLARE_MODULE(sys_comp_google_ctc_audio_processing_init);
+SOF_MODULE_INIT(google_ctc_audio_processing,
+		sys_comp_google_ctc_audio_processing_init);
diff --git a/src/audio/google/google_ctc_audio_processing_mock.c b/src/audio/google/google_ctc_audio_processing_mock.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Copyright(c) 2024 Google LLC.
+//
+// Author: Eddy Hsu <[email protected]>
+#include "google_ctc_audio_processing.h"
+
+#include <rtos/alloc.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ipc/topology.h"
+
+struct GoogleCtcAudioProcessingState {
+	int num_frames;
+	int partition_size;
+	int impulse_size;
+	int chunk_frames;
+	int sample_rate;
+	int is_symmetric;
+};
+
+GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void)
+{
+	struct GoogleCtcAudioProcessingState *s =
+		rballoc(0, SOF_MEM_CAPS_RAM, sizeof(GoogleCtcAudioProcessingState));
+	if (!s)
+		return NULL;
+
+	s->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES;
+	s->partition_size = 64;
+	s->impulse_size = 256;
+	s->chunk_frames = 480;
+	s->sample_rate = 48000;
+	s->is_symmetric = 0;
+	return s;
+}
+
+void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state)
+{
+	if (state)
+		rfree(state);
+}
+
+int GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *const state,
+				    const int16_t *const src,
+				    int16_t *const dest)
+{
+	memcpy_s(dest, src, sizeof(int16_t) * state->num_frames);
+	return 0;
+}