From 2cc2bad448ba8377a9eb7e26b38d7d988fd2cce6 Mon Sep 17 00:00:00 2001 From: Eddy Hsu Date: Wed, 3 Jan 2024 11:05:32 -0800 Subject: [PATCH] [WIP] comp: Add initial Google CTC component Introduce a new component to perform crosstalk-cancellation on the capture path. Signed-off-by: Eddy Hsu --- src/audio/google/CMakeLists.txt | 20 ++ src/audio/google/Kconfig | 26 ++ .../google/google_ctc_audio_processing.c | 277 ++++++++++++++++++ .../google/google_ctc_audio_processing_mock.c | 53 ++++ .../include/google_ctc_audio_processing.h | 34 +++ .../platform/intel/google-ctc-reference.conf | 41 +++ 6 files changed, 451 insertions(+) create mode 100644 src/audio/google/google_ctc_audio_processing.c create mode 100644 src/audio/google/google_ctc_audio_processing_mock.c create mode 100644 third_party/include/google_ctc_audio_processing.h create mode 100644 tools/topology/topology2/platform/intel/google-ctc-reference.conf diff --git a/src/audio/google/CMakeLists.txt b/src/audio/google/CMakeLists.txt index 3936c8041b67..b6f8d3725cd8 100644 --- a/src/audio/google/CMakeLists.txt +++ b/src/audio/google/CMakeLists.txt @@ -29,6 +29,26 @@ if((NOT CONFIG_LIBRARY) OR CONFIG_LIBRARY_STATIC) target_link_libraries(sof PRIVATE c) endif() endif() + + if(CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING) + target_include_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/include) + add_local_sources(sof + google_ctc_audio_processing.c + ) + if(CONFIG_GOOGLE_CTC_AUDIO_PROCESSING_MOCK) + add_local_sources(sof + google_ctc_audio_processing_mock.c + ) + else() + message(INFO "Link with google_ctc_audio_processing") + target_link_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/lib) + target_link_libraries(sof PRIVATE google_ctc_audio_processing) + target_link_libraries(sof PRIVATE c++) + target_link_libraries(sof PRIVATE c++abi) + target_link_libraries(sof PRIVATE m) + target_link_libraries(sof PRIVATE c) + endif() + endif() return() endif() diff --git a/src/audio/google/Kconfig b/src/audio/google/Kconfig index 027e15a2c229..1e60d3487b5e 100644 --- a/src/audio/google/Kconfig +++ b/src/audio/google/Kconfig @@ -80,4 +80,30 @@ config GOOGLE_RTC_AUDIO_PROCESSING_MOCK Mock Google real-time communication audio processing. It allows for compilation check and basic audio flow checking. +config COMP_GOOGLE_CTC_AUDIO_PROCESSING + bool "Google Crosstalk Cancellation Audio processing" + select COMP_BLOB + select GOOGLE_CTC_AUDIO_PROCESSING_MOCK if COMP_STUBS + default n + help + Select for Google crosstalk cancellation audio processing. It + uses the Google real-time audio processing library to perform + crosstalk cancellation. + +config COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES + depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING + int "Number of frames to process for Google Crosstalk Cancellation Audio processing" + default 64 + help + Sets the number of frames to process in the Google crosstalk + cancellation audio processing. + +config GOOGLE_CTC_AUDIO_PROCESSING_MOCK + bool "Google Crosstalk Cancellation Audio processing mock" + default n + depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING + help + Mock Google crosstalk cancellation audio processing. + It allows for compilation check and basic audio flow checking. + endmenu diff --git a/src/audio/google/google_ctc_audio_processing.c b/src/audio/google/google_ctc_audio_processing.c new file mode 100644 index 000000000000..d7c30ce66efa --- /dev/null +++ b/src/audio/google/google_ctc_audio_processing.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2024 Google LLC. +// +// Author: Eddy Hsu +#include +#include +#include +#include + +#include + +struct comp_driver comp_ctc_audio_processing; + +/* bf0e1bbc-dc6a-45fe-bc90-2554cb137ab4 */ +DECLARE_SOF_RT_UUID("google-ctc-audio-processing", google_ctc_audio_processing_uuid, + 0xbf0e1bbc, 0xdc6a, 0x45fe, 0xbc, 0x90, 0x25, 0x54, 0xcb, + 0x13, 0x7a, 0xb4); + +DECLARE_TR_CTX(google_ctc_audio_processing_tr, SOF_UUID(google_ctc_audio_processing_uuid), + LOG_LEVEL_INFO); + +struct google_ctc_audio_processing_comp_data { + struct comp_data_blob_handler *model_handler; + struct comp_buffer *input; + struct comp_buffer *output; + uint32_t num_frames; + GoogleCtcAudioProcessingState *state; + struct comp_data_blob_handler *tuning_handler; + bool reconfigure; +}; + +struct comp_driver comp_ctc_audio_processing = { + .uid = SOF_RT_UUID(google_ctc_audio_processing_uuid), + .tctx = &google_ctc_audio_processing_tr, + .ops = { + .create = ctc_audio_processing_create, + .free = ctc_audio_processing_free, + .params = NULL, + .cmd = ctc_audio_processing_cmd, + .trigger = ctc_audio_processing_trigger, + .prepare = ctc_audio_processing_prepare, + .reset = ctc_audio_processing_reset, + .copy = ctc_audio_processing_copy, + }, +}; + +static SHARED_DATA struct comp_driver_info comp_ctc_audio_processing_info = { + .drv = &comp_ctc_audio_processing, +}; + +static void sys_comp_ctc_audio_processing_init(void) +{ + comp_register(platform_shared_get(&comp_ctc_audio_processing_info, + sizeof(comp_ctc_audio_processing_info))); +} + +static struct comp_dev *ctc_audio_processing_create(const struct comp_driver *drv, + struct comp_ipc_config *config, + void *spec) +{ + struct comp_dev *dev; + struct ctc_audio_processing_comp_data *cd; + struct ipc_config_process *ipc_ctc = spec; + + comp_cl_info(&comp_ctc_audio_processing, "ctc_audio_processing_create()"); + + dev = comp_alloc(drv, sizeof(*dev)); + if (!dev) + return NULL; + dev->ipc_config = *config; + + cd = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*cd)); + if (!cd) { + rfree(dev); + return NULL; + } + + cd->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES; + + cd->input = + rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->input[0])); + if (!cd->input) + goto fail; + bzero(cd->input, cd->num_frames * sizeof(cd->input[0])); + + cd->output = + rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->output[0])); + if (!cd->output) + goto fail; + bzero(cd->output, cd->num_frames * sizeof(cd->output[0])); + + cd->state = GoogleCtcAudioProcessingCreate(); + + comp_set_drvdata(dev, cd); + + dev->state = COMP_STATE_READY; + + comp_dbg(dev, "crosstalk cancellation created"); + + return dev; +fail: + comp_err(dev, "google_ctc_audio_processing_create(): Failed"); + if (cd) { + rfree(cd->input); + rfree(cd->output); + if (cd->state) + GoogleCtcAudioProcessingFree(cd->state); + rfree(cd); + } + rfree(dev); + return NULL; +} + +static void ctc_audio_processing_free(struct comp_dev *dev) +{ + struct comp_data *cd = comp_get_drvdata(dev); + + GoogleCtcAudioProcessingFree(cd->state); + cd->state = NULL; + rfree(cd->input); + rfree(cd->output); + rfree(cd); + rfree(dev); +} + +static int ctc_audio_processing_trigger(struct comp_dev *dev, int cmd) +{ + comp_dbg(dev, "crosstalk cancellation got trigger cmd %d", cmd); + return comp_set_state(dev, cmd); +} + +static int ctc_audio_processing_prepare(struct comp_dev *dev) +{ + int ret; + + comp_info(dev, "google_ctc_audio_processing_prepare()"); + + ret = comp_set_state(dev, COMP_TRIGGER_PREPARE); + if (ret < 0) + return ret; + + if (ret == COMP_STATUS_STATE_ALREADY_SET) + return PPL_STATUS_PATH_STOP; + + comp_dbg(dev, "crosstalk cancellation prepared"); + return 0; +} + +static int ctc_audio_processing_reset(struct comp_dev *dev) +{ + return comp_set_state(dev, COMP_TRIGGER_RESET); +} + +static int ctc_audio_processing_copy(struct comp_dev *dev) +{ + struct comp_copy_limits cl; + struct comp_buffer *source; + struct comp_buffer *sink; + struct ctc_audio_processing_comp_data *cd = comp_get_drvdata(dev); + int frame; + int channel; + uint32_t buff_frag = 0; + int16_t *src; + int16_t *dst; + + comp_dbg(dev, "ctc_audio_processing_copy()"); + + source = list_first_item(&dev->bsource_list, struct comp_buffer, sink_list); + sink = list_first_item(&dev->bsink_list, struct comp_buffer, source_list); + + comp_get_copy_limits_with_lock(source, sink, &cl); + + buffer_invalidate(source, cl.source_bytes); + + for (frame = 0; frame < cl.frames; frame++) { + for (channel = 0; channel < sink->stream.channels; channel++) { + src = audio_stream_read_frag_s16(&source->stream, buff_frag); + dst = audio_stream_write_frag_s16(&sink->stream, buff_frag); + + GoogleCtcAudioProcessingProcess(cd->state, src, dst); + + ++buff_frag; + } + } + + buffer_writeback(sink, cl.sink_bytes); + + comp_update_buffer_produce(sink, cl.sink_bytes); + comp_update_buffer_consume(source, cl.source_bytes); + + return 0; +} + +static int ctc_audio_processing_cmd_set_data(struct comp_dev *dev, + struct sof_ipc_ctrl_data *cdata) +{ + struct ctc_audio_processing_comp_data *cd = comp_get_drvdata(dev); + + if (cdata->cmd != SOF_CTRL_CMD_BINARY) { + comp_err(dev, "ctc_audio_processing_cmd_set_data(): invalid cmd %d", cdata->cmd); + return -EINVAL; + } + + ret = comp_data_blob_set_cmd(cd->tuning_handler, cdata); + if (ret) + return ret; + + /* Accept the new blob immediately so that userspace can write + * the control in quick succession without error. + * This ensures the last successful control write from userspace + * before prepare/copy is applied. + * The config blob is not referenced after reconfigure() returns + * so it is safe to call comp_get_data_blob here which frees the + * old blob. This assumes cmd() and prepare()/copy() cannot run + * concurrently which is the case when there is no preemption. + */ + if (comp_is_new_data_blob_available(cd->tuning_handler)) { + comp_get_data_blob(cd->tuning_handler, NULL, NULL); + cd->reconfigure = true; + } + + comp_dbg(dev, "crosstalk cancellation new settings"); + return 0; +} + +static int ctc_audio_processing_cmd_get_data(struct comp_dev *dev, + struct sof_ipc_ctrl_data *cdata, + int max_data_size) +{ + struct ctc_audio_processing_comp_data *cd = comp_get_drvdata(dev); + + if (cdata->cmd != SOF_CTRL_CMD_BINARY) { + comp_err(dev, "ctc_audio_processing_cmd_get_data(): invalid cmd %d", cdata->cmd); + return -EINVAL; + } + + return comp_data_blob_get_cmd(cd->tuning_handler, cdata, max_data_size); +} + +static int ctc_audio_processing_cmd(struct comp_dev *dev, int cmd, void *data, + int max_data_size) +{ + struct sof_ipc_ctrl_data *cdata = data; + int ret = 0; + + switch (cmd) { + case COMP_CMD_SET_VALUE: + case COMP_CMD_GET_VALUE: + return 0; + case COMP_CMD_SET_DATA: + ret = ctc_audio_processing_cmd_set_data(dev, cdata); + break; + case COMP_CMD_GET_DATA: + ret = ctc_audio_processing_cmd_get_data(dev, cdata, max_data_size); + break; + default: + comp_err(dev, "ctc_audio_processing_cmd(): unhandled command %d", cmd); + ret = -EINVAL; + break; + } + return ret; +} + +static SHARED_DATA struct comp_driver_info google_ctc_audio_processing_info = { + .drv = &google_ctc_audio_processing, +}; + +UT_STATIC void sys_comp_google_ctc_audio_processing_init(void) +{ + comp_register(platform_shared_get(&google_ctc_audio_processing_info, + sizeof(google_ctc_audio_processing_info))); +} + +DECLARE_MODULE(sys_comp_google_ctc_audio_processing_init); +SOF_MODULE_INIT(google_ctc_audio_processing, + sys_comp_google_ctc_audio_processing_init); diff --git a/src/audio/google/google_ctc_audio_processing_mock.c b/src/audio/google/google_ctc_audio_processing_mock.c new file mode 100644 index 000000000000..2c3f55b06096 --- /dev/null +++ b/src/audio/google/google_ctc_audio_processing_mock.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2024 Google LLC. +// +// Author: Eddy Hsu +#include "google_ctc_audio_processing.h" + +#include +#include +#include +#include + +#include "ipc/topology.h" + +struct GoogleCtcAudioProcessingState { + int num_frames; + int partition_size; + int impulse_size; + int chunk_frames; + int sample_rate; + int is_symmetric; +}; + +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void) +{ + struct GoogleCtcAudioProcessingState *s = + rballoc(0, SOF_MEM_CAPS_RAM, sizeof(GoogleCtcAudioProcessingState)); + if (!s) + return NULL; + + s->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES; + s->partition_size = 64; + s->impulse_size = 256; + s->chunk_frames = 480; + s->sample_rate = 48000; + s->is_symmetric = 0; + return s; +} + +void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state) +{ + if (state) + rfree(state); +} + +int GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *const state, + const int16_t *const src, + int16_t *const dest) +{ + memcpy_s(dest, sizeof(int16_t) * state->num_frames, + src, sizeof(int16_t) * state->num_frames); + return 0; +} diff --git a/third_party/include/google_ctc_audio_processing.h b/third_party/include/google_ctc_audio_processing.h new file mode 100644 index 000000000000..1f584fbb122f --- /dev/null +++ b/third_party/include/google_ctc_audio_processing.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright(c) 2024 Google LLC. + * + * Author: Eddy Hsu + */ +#ifndef GOOGLE_CTC_AUDIO_PROCESSING_H +#define GOOGLE_CTC_AUDIO_PROCESSING_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct GoogleCtcAudioProcessingState GoogleCtcAudioProcessingState; + +// Creates an instance of GoogleCtcAudioProcessing with the tuning embedded in +// the library. If creation fails, NULL is returned. +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void); + +// Frees all allocated resources in `state`. +void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state); + +// Apply CTC on `src` and produce result in `dest`. +int GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *const state, + const int16_t *const src, + int16_t *const dest); + +#ifdef __cplusplus +} +#endif + +#endif // GOOGLE_CTC_AUDIO_PROCESSING_H diff --git a/tools/topology/topology2/platform/intel/google-ctc-reference.conf b/tools/topology/topology2/platform/intel/google-ctc-reference.conf new file mode 100644 index 000000000000..8d887c9febe8 --- /dev/null +++ b/tools/topology/topology2/platform/intel/google-ctc-reference.conf @@ -0,0 +1,41 @@ +Object.Pipeline.google-ctc-capture [ + { + index 18 # TODO: update index + core_id $GOOGLE_CTC_CORE_ID + + Object.Widget.pipeline.1 { + stream_name "DMIC0 CTC" + core $GOOGLE_CTC_CORE_ID + priority $GOOGLE_CTC_PIPELINE_PRIORITY + } + + Object.Widget.google-ctc.1 { + Object.Base.input_pin_binding.1 { + input_pin_binding_name "module-copier.18.1" + } + + Object.Base.input_pin_binding.2 { + input_pin_binding_name "dai-copier.SSP.$SPEAKER_CODEC_NAME.capture" + } + } + } +] + +Object.Base.route [ + { + source $DMIC0_DAI_PIPELINE_SRC + sink module-copier.18.1 + } + { + source "dai-copier.SSP.$SPEAKER_CODEC_NAME.capture" + sink "google-ctc.18.1" + } + { + source "module-copier.18.2" + sink "host-copier.27.capture" + } + { + source "gain.20.1" + sink "host-copier.28.capture" + } +]