Optimized kernels for Tensorflow Lite for MCU

[Yaxit]

Is your enhancement proposal related to a problem? Please describe.
Tensorflow lite for Microcontrollers supports optimized kernels to accelerate the inference (eg: CMSIS NN).
Currently I could not find a way to configure in zephyr whether to use the kernels or not, and it seems it falls back to the unoptimized implementation.

Describe the solution you'd like
Add CONFIG_ settings to enable the optimized kernels. Possibly automate the selection based on the target platform.

[geekbozu]

I'm in the same boat where I could really use the optimized kernels, but TFL-M is very behind, as well as there is no clear way to use these optimized kernels. Getting any models to not just crash with Zephyr has been...tricky.

I'll keep poking at it to see if I can make it work but some guidance here would be cool.

[Yaxit]

I agree. In my case I got it working with a workaround by modifying the CMake file of the library, see below.
I believe integrating these changes should be feasible, maybe the sketchy part is that this makes the module for tflite depend on cmsis module.

Hope this helps.

zephyr\modules\tflite-micro\CMakeLists.txt

# Copyright (c) 2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

if(CONFIG_TENSORFLOW_LITE_MICRO)

  set(TENSORFLOW_LITE_MICRO_DIR ${ZEPHYR_CURRENT_MODULE_DIR})

  zephyr_library()

  zephyr_include_directories(
    ${TENSORFLOW_LITE_MICRO_DIR}/../../hal/cmsis
    ${TENSORFLOW_LITE_MICRO_DIR}/.
    ${TENSORFLOW_LITE_MICRO_DIR}/third_party_static/gemmlowp
    ${TENSORFLOW_LITE_MICRO_DIR}/third_party_static/flatbuffers/include
    ${TENSORFLOW_LITE_MICRO_DIR}/third_party_static/ruy
  )

  zephyr_library_sources(
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/simple_memory_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/debug_log.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_error_reporter.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/all_ops_resolver.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/memory_helpers.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/test_helpers.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/recording_micro_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_time.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/recording_simple_memory_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_string.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_profiler.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/flatbuffer_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_graph.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/mock_micro_graph.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_interpreter.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/system_setup.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/memory_planner/linear_memory_planner.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/c/common.c
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/kernels/internal/quantization_util.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/kernels/kernel_util.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/flatbuffer_conversions.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/error_reporter.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/tensor_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/op_resolver.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/schema/schema_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/activations.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/activations_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/add.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/add_n.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/arg_min_max.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/batch_to_space_nd.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cast.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/ceil.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/circular_buffer.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/comparisons.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/concatenation.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/conv.cc # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/conv_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cumsum.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/depth_to_space.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/depthwise_conv.cc # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/depthwise_conv_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/dequantize.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/detection_postprocess.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/elementwise.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/elu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/ethosu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/exp.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/expand_dims.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/fill.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/floor.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/floor_div.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/floor_mod.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/fully_connected.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/fully_connected_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/gather.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/gather_nd.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/hard_swish.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/hard_swish_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/if.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/kernel_runner.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/kernel_util.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/l2norm.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/l2_pool_2d.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/leaky_relu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logical.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logical_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logistic.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logistic_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/log_softmax.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/maximum_minimum.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/mul.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/neg.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/pack.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/pad.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/pooling.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/pooling_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/prelu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/quantize.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/quantize_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/reduce.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/reshape.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/resize_bilinear.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/round.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/shape.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/softmax.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/softmax_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/space_to_batch_nd.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/space_to_depth.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/split.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/split_v.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/squeeze.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/strided_slice.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/sub.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/svdf.cc # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/svdf_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/tanh.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/transpose.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/transpose_conv.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/unpack.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/zeros_like.cc
  )

endif()

[geekbozu]

It builds now. My model just seems to be made with to new of a version of tensorflow as it just runs into an exit condition on invoke...
Thanks for the help! Its a start :D

[stephanosio]

FYI @JordanYates

[Yaxit]

Should have been closed with the new module structure
#56957