From b0a5f0c19bb5a96fb9312689cd79cf7704dd4f68 Mon Sep 17 00:00:00 2001 From: Eddy Hsu Date: Wed, 31 Jan 2024 15:03:49 -0800 Subject: [PATCH] comp: Add initial Google CTC component Introduce a new component to perform crosstalk-cancellation on the playback path. Signed-off-by: Eddy Hsu --- src/audio/google/CMakeLists.txt | 20 + src/audio/google/Kconfig | 26 ++ .../google/google_ctc_audio_processing.c | 359 ++++++++++++++++++ .../google/google_ctc_audio_processing_mock.c | 53 +++ .../include/google_ctc_audio_processing.h | 44 +++ 5 files changed, 502 insertions(+) create mode 100644 src/audio/google/google_ctc_audio_processing.c create mode 100644 src/audio/google/google_ctc_audio_processing_mock.c create mode 100644 third_party/include/google_ctc_audio_processing.h diff --git a/src/audio/google/CMakeLists.txt b/src/audio/google/CMakeLists.txt index 3936c8041b67..b6f8d3725cd8 100644 --- a/src/audio/google/CMakeLists.txt +++ b/src/audio/google/CMakeLists.txt @@ -29,6 +29,26 @@ if((NOT CONFIG_LIBRARY) OR CONFIG_LIBRARY_STATIC) target_link_libraries(sof PRIVATE c) endif() endif() + + if(CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING) + target_include_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/include) + add_local_sources(sof + google_ctc_audio_processing.c + ) + if(CONFIG_GOOGLE_CTC_AUDIO_PROCESSING_MOCK) + add_local_sources(sof + google_ctc_audio_processing_mock.c + ) + else() + message(INFO "Link with google_ctc_audio_processing") + target_link_directories(sof PRIVATE ${CMAKE_SOURCE_DIR}/third_party/lib) + target_link_libraries(sof PRIVATE google_ctc_audio_processing) + target_link_libraries(sof PRIVATE c++) + target_link_libraries(sof PRIVATE c++abi) + target_link_libraries(sof PRIVATE m) + target_link_libraries(sof PRIVATE c) + endif() + endif() return() endif() diff --git a/src/audio/google/Kconfig b/src/audio/google/Kconfig index 027e15a2c229..1e60d3487b5e 100644 --- a/src/audio/google/Kconfig +++ b/src/audio/google/Kconfig @@ -80,4 +80,30 @@ config GOOGLE_RTC_AUDIO_PROCESSING_MOCK Mock Google real-time communication audio processing. It allows for compilation check and basic audio flow checking. +config COMP_GOOGLE_CTC_AUDIO_PROCESSING + bool "Google Crosstalk Cancellation Audio processing" + select COMP_BLOB + select GOOGLE_CTC_AUDIO_PROCESSING_MOCK if COMP_STUBS + default n + help + Select for Google crosstalk cancellation audio processing. It + uses the Google real-time audio processing library to perform + crosstalk cancellation. + +config COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES + depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING + int "Number of frames to process for Google Crosstalk Cancellation Audio processing" + default 64 + help + Sets the number of frames to process in the Google crosstalk + cancellation audio processing. + +config GOOGLE_CTC_AUDIO_PROCESSING_MOCK + bool "Google Crosstalk Cancellation Audio processing mock" + default n + depends on COMP_GOOGLE_CTC_AUDIO_PROCESSING + help + Mock Google crosstalk cancellation audio processing. + It allows for compilation check and basic audio flow checking. + endmenu diff --git a/src/audio/google/google_ctc_audio_processing.c b/src/audio/google/google_ctc_audio_processing.c new file mode 100644 index 000000000000..7931d0608d63 --- /dev/null +++ b/src/audio/google/google_ctc_audio_processing.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2024 Google LLC. +// +// Author: Eddy Hsu +#include +#include +#include +#include + +#include + +LOG_MODULE_REGISTER(google_ctc_audio_processing, CONFIG_SOF_LOG_LEVEL); + +/* bf0e1bbc-dc6a-45fe-bc90-2554cb137ab4 */ +DECLARE_SOF_RT_UUID("google-ctc-audio-processing", google_ctc_audio_processing_uuid, + 0xbf0e1bbc, 0xdc6a, 0x45fe, 0xbc, 0x90, 0x25, 0x54, 0xcb, + 0x13, 0x7a, 0xb4); + +DECLARE_TR_CTX(google_ctc_audio_processing_tr, SOF_UUID(google_ctc_audio_processing_uuid), + LOG_LEVEL_INFO); + +struct google_ctc_audio_processing_comp_data { + struct comp_buffer *input; + struct comp_buffer *output; + uint32_t num_frames; + GoogleCtcAudioProcessingState *state; + struct comp_data_blob_handler *tuning_handler; + bool reconfigure; +}; + +static int ctc_free(struct processing_module *mod) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + + if (cd) { + rfree(cd->input); + rfree(cd->output); + if (cd->state) { + GoogleCtcAudioProcessingFree(cd->state); + cd->state = NULL; + } + rfree(cd); + } + + return 0; +} + +static int ctc_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct google_ctc_audio_processing_comp_data *cd; + int ret; + + comp_info(dev, "ctc_init()"); + + /* Create private component data */ + cd = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*cd)); + if (!cd) { + ret = -ENOMEM; + goto fail; + } + + md->private = cd; + + cd->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES; + + cd->input = + rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->input[0])); + if (!cd->input) { + ret = -ENOMEM; + goto fail; + } + + cd->output = + rballoc(0, SOF_MEM_CAPS_RAM, cd->num_frames * sizeof(cd->output[0])); + if (!cd->output) { + ret = -ENOMEM; + goto fail; + } + + cd->state = GoogleCtcAudioProcessingCreateWithConfig( + cd->num_frames, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_PARTITION_SIZE, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_IMPULSE_SIZE, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_CHUNK_FRAMES, + CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_SAMPLE_RATE, + /*config=*/NULL, /*config_size=*/0); + + comp_dbg(dev, "ctc_init(): Ready"); + + return 0; +fail: + comp_err(dev, "ctc_init(): Failed"); + ctc_free(mod); + return ret; +} + +static int google_ctc_audio_processing_reconfigure(struct processing_module *mod) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + uint8_t *config; + size_t size; + int ret; + + comp_dbg(dev, "google_ctc_audio_processing_reconfigure()"); + + if (!comp_is_current_data_blob_valid(cd->tuning_handler) && + !comp_is_new_data_blob_available(cd->tuning_handler)) { + /* + * The data blob hasn't been available once so far. + * + * This looks redundant since the same check will be done in + * comp_get_data_blob() below. But without this early return, + * hundreds of warn message lines are produced per second by + * comp_get_data_blob() calls until the data blob is arrived. + */ + return 0; + } + + config = comp_get_data_blob(cd->tuning_handler, &size, NULL); + if (size == 0) { + /* No data to be handled */ + return 0; + } + + if (!config) { + comp_err(dev, "google_ctc_audio_processing_reconfigure(): Tuning config not set"); + return -EINVAL; + } + + comp_info(dev, "google_ctc_audio_processing_reconfigure(): New tuning config %p (%zu bytes)", + config, size); + + cd->reconfigure = false; + + uint8_t *google_ctc_audio_processing_config; + size_t google_ctc_audio_processing_config_size; + bool google_ctc_audio_processing_config_present; + + GoogleCtcAudioProcessingParseSofConfigMessage(config, size, + &google_ctc_audio_processing_config, + &google_ctc_audio_processing_config_size, + &google_ctc_audio_processing_config_present); + + if (google_ctc_audio_processing_config_present) { + comp_info(dev, + "google_ctc_audio_processing_reconfigure(): Applying config of size %zu bytes", + google_ctc_audio_processing_config_size); + + ret = GoogleCtcAudioProcessingReconfigure(cd->state, + google_ctc_audio_processing_config, + google_ctc_audio_processing_config_size); + if (ret) { + comp_err(dev, "GoogleCtcAudioProcessingReconfigure failed: %d", + ret); + return ret; + } + } + + return 0; +} + +static int ctc_prepare(struct processing_module *mod) +{ + comp_info(mod->dev, "ctc_prepare()"); + + return 0; +} + +static int ctc_reset(struct processing_module *mod) +{ + comp_dbg(mod->dev, "ctc_reset()"); + + return 0; +} + +static int ctc_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + + int16_t const *in; + int8_t const *in_buf_start; + int8_t const *in_buf_end; + size_t in_buf_size; + + int16_t *out; + int8_t *out_buf_start; + int8_t *out_buf_end; + size_t out_buf_size; + + size_t num_of_bytes_to_process; + + struct sof_source *in_stream; + struct sof_sink *out_stream; + + comp_dbg(dev, "ctc_process()"); + + if (cd->reconfigure) { + ret = google_ctc_audio_processing_reconfigure(mod); + if (ret) + return ret; + } + + in_stream = sources[0]; + out_stream = sinks[0]; + + num_of_bytes_to_process = cd->num_frames * source_get_frame_bytes(in_stream); + + ret = source_get_data(in_stream, num_of_bytes_to_process, (const void **)&in, + (const void **)&in_buf_start, &in_buf_size); + assert(!ret); + + ret = sink_get_buffer(out_stream, num_of_bytes_to_process, (void **)&out, + (void **)&out_buf_start, &out_buf_size); + assert(!ret); + + GoogleCtcAudioProcessingProcess(cd->state, in, out, cd->num_frames); + + source_release_data(in_stream, num_of_bytes_to_process); + sink_commit_buffer(out_stream, num_of_bytes_to_process); + + return 0; +} + +#if CONFIG_IPC_MAJOR_3 +static int ctc_cmd_set_data(struct processing_module *mod, + struct sof_ipc_ctrl_data *cdata) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + int ret; + + switch (cdata->cmd) { + case SOF_CTRL_CMD_BINARY: + ret = comp_data_blob_set_cmd(cd->tuning_handler, cdata); + if (ret) + return ret; + /* Accept the new blob immediately so that userspace can write + * the control in quick succession without error. + * This ensures the last successful control write from userspace + * before prepare/copy is applied. + * The config blob is not referenced after reconfigure() returns + * so it is safe to call comp_get_data_blob here which frees the + * old blob. This assumes cmd() and prepare()/copy() cannot run + * concurrently which is the case when there is no preemption. + */ + if (comp_is_new_data_blob_available(cd->tuning_handler)) { + comp_get_data_blob(cd->tuning_handler, NULL, NULL); + cd->reconfigure = true; + } + return 0; + default: + comp_err(mod->dev, + "google_ctc_audio_processing_ctrl_set_data(): Only binary controls supported %d", + cdata->cmd); + return -EINVAL; + } +} + +static int ctc_cmd_get_data(struct processing_module *mod, + struct sof_ipc_ctrl_data *cdata, + size_t max_data_size) +{ + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + + comp_info(mod->dev, "google_ctc_audio_processing_ctrl_get_data(): %u", cdata->cmd); + + switch (cdata->cmd) { + case SOF_CTRL_CMD_BINARY: + return comp_data_blob_get_cmd(cd->tuning_handler, cdata, max_data_size); + default: + comp_err(mod->dev, + "google_ctc_audio_processing_ctrl_get_data(): Only binary controls supported %d", + cdata->cmd); + return -EINVAL; + } +} +#endif + +static int ctc_set_config(struct processing_module *mod, uint32_t param_id, + enum module_cfg_fragment_position pos, + uint32_t data_offset_size, + const uint8_t *fragment, + size_t fragment_size, uint8_t *response, + size_t response_size) +{ +#if CONFIG_IPC_MAJOR_4 + struct google_ctc_audio_processing_comp_data *cd = module_get_private_data(mod); + int ret; + + switch (param_id) { + case SOF_IPC4_SWITCH_CONTROL_PARAM_ID: + case SOF_IPC4_ENUM_CONTROL_PARAM_ID: + comp_err(mod->dev, "google_ctc_audio_processing_set_data(): Only binary controls supported"); + return -EINVAL; + } + + ret = comp_data_blob_set(cd->tuning_handler, pos, data_offset_size, + fragment, fragment_size); + if (ret) + return ret; + + /* Accept the new blob immediately so that userspace can write + * the control in quick succession without error. + * This ensures the last successful control write from userspace + * before prepare/copy is applied. + * The config blob is not referenced after reconfigure() returns + * so it is safe to call comp_get_data_blob here which frees the + * old blob. This assumes cmd() and prepare()/copy() cannot run + * concurrently which is the case when there is no preemption. + * + * Note from review: A race condition is possible and should be + * further investigated and fixed. + */ + if (comp_is_new_data_blob_available(cd->tuning_handler)) { + comp_get_data_blob(cd->tuning_handler, NULL, NULL); + cd->reconfigure = true; + } + + return 0; +#elif CONFIG_IPC_MAJOR_3 + struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; + + return ctc_cmd_set_data(mod, cdata); +#endif +} + +static int ctc_get_config(struct processing_module *mod, + uint32_t param_id, uint32_t *data_offset_size, + uint8_t *fragment, size_t fragment_size) +{ +#if CONFIG_IPC_MAJOR_4 + comp_err(mod->dev, "ctc_get_config(): Not supported"); + return -EINVAL; +#elif CONFIG_IPC_MAJOR_3 + struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; + + return ctc_cmd_get_data(mod, cdata, fragment_size); +#endif +} + +static struct module_interface google_ctc_audio_processing_interface = { + .init = ctc_init, + .free = ctc_free, + .process_audio_stream = ctc_process, + .prepare = ctc_prepare, + .set_configuration = ctc_set_config, + .get_configuration = ctc_get_config, + .reset = ctc_reset, +}; + +DECLARE_MODULE_ADAPTER(google_ctc_audio_processing_interface, + google_ctc_audio_processing_uuid, google_ctc_audio_processing_tr); +SOF_MODULE_INIT(google_ctc_audio_processing, + sys_comp_module_google_ctc_audio_processing_interface_init); diff --git a/src/audio/google/google_ctc_audio_processing_mock.c b/src/audio/google/google_ctc_audio_processing_mock.c new file mode 100644 index 000000000000..721637bc4c72 --- /dev/null +++ b/src/audio/google/google_ctc_audio_processing_mock.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2024 Google LLC. +// +// Author: Eddy Hsu +#include "google_ctc_audio_processing.h" + +#include +#include +#include +#include + +#include "ipc/topology.h" + +struct GoogleCtcAudioProcessingState { + int num_frames; + int partition_size; + int impulse_size; + int chunk_frames; + int sample_rate; + int is_symmetric; +}; + +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void) +{ + struct GoogleCtcAudioProcessingState *s = + rballoc(0, SOF_MEM_CAPS_RAM, sizeof(GoogleCtcAudioProcessingState)); + if (!s) + return NULL; + + s->num_frames = CONFIG_COMP_GOOGLE_CTC_AUDIO_PROCESSING_NUM_FRAMES; + s->partition_size = 64; + s->impulse_size = 256; + s->chunk_frames = 480; + s->sample_rate = 48000; + s->is_symmetric = 0; + return s; +} + +void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state) +{ + if (state) + rfree(state); +} + +int GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *state, + const int16_t *src, int16_t *dest, + size_t num_frames) +{ + memcpy_s(dest, sizeof(int16_t) * num_frames, + src, sizeof(int16_t) * num_frames); + return 0; +} diff --git a/third_party/include/google_ctc_audio_processing.h b/third_party/include/google_ctc_audio_processing.h new file mode 100644 index 000000000000..95fd51f52f7b --- /dev/null +++ b/third_party/include/google_ctc_audio_processing.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright(c) 2024 Google LLC. + * + * Author: Eddy Hsu + */ +#ifndef GOOGLE_CTC_AUDIO_PROCESSING_H +#define GOOGLE_CTC_AUDIO_PROCESSING_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct GoogleCtcAudioProcessingState GoogleCtcAudioProcessingState; + +// Creates an instance of GoogleCtcAudioProcessing with the tuning embedded in +// the library. If creation fails, NULL is returned. +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreate(void); + +GoogleCtcAudioProcessingState *GoogleCtcAudioProcessingCreateWithConfig( + int num_frames, int partition_size, int impulse_size, int chunk_frames, + int sample_rate, int is_symmetric, const uint8_t *config, int config_size); + +// Frees all allocated resources in `state`. +void GoogleCtcAudioProcessingFree(GoogleCtcAudioProcessingState *state); + +// Apply CTC on `src` and produce result in `dest`. +void GoogleCtcAudioProcessingProcess(GoogleCtcAudioProcessingState *state, + const float *src, float *dest, + int num_frames); + + +// Reconfigure the audio processing. +// Returns 0 if success and non zero if failure. +int GoogleCtcAudioProcessingReconfigure(GoogleCtcAudioProcessingState *state, + const uint8_t *config, int config_size); + +#ifdef __cplusplus +} +#endif + +#endif // GOOGLE_CTC_AUDIO_PROCESSING_H