search

zephyrproject-rtos / zephyr

Primary Git Repository for the Zephyr Project. Zephyr is a new generation, scalable, optimized, secure RTOS for multiple hardware architectures.
https://docs.zephyrproject.org
Apache License 2.0
10.68k stars 6.53k forks source link

Optimized kernels for Tensorflow Lite for MCU #45825

Closed Yaxit closed 9 months ago

Yaxit commented 2 years ago

Is your enhancement proposal related to a problem? Please describe. Tensorflow lite for Microcontrollers supports optimized kernels to accelerate the inference (eg: CMSIS NN). Currently I could not find a way to configure in zephyr whether to use the kernels or not, and it seems it falls back to the unoptimized implementation.

Describe the solution you'd like Add CONFIG_ settings to enable the optimized kernels. Possibly automate the selection based on the target platform.

geekbozu commented 2 years ago

I'm in the same boat where I could really use the optimized kernels, but TFL-M is very behind, as well as there is no clear way to use these optimized kernels. Getting any models to not just crash with Zephyr has been...tricky.

I'll keep poking at it to see if I can make it work but some guidance here would be cool.

Yaxit commented 2 years ago

I agree. In my case I got it working with a workaround by modifying the CMake file of the library, see below. I believe integrating these changes should be feasible, maybe the sketchy part is that this makes the module for tflite depend on cmsis module.

Hope this helps.

zephyr\modules\tflite-micro\CMakeLists.txt

# Copyright (c) 2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

if(CONFIG_TENSORFLOW_LITE_MICRO)

  set(TENSORFLOW_LITE_MICRO_DIR ${ZEPHYR_CURRENT_MODULE_DIR})

  zephyr_library()

  zephyr_include_directories(
    ${TENSORFLOW_LITE_MICRO_DIR}/../../hal/cmsis
    ${TENSORFLOW_LITE_MICRO_DIR}/.
    ${TENSORFLOW_LITE_MICRO_DIR}/third_party_static/gemmlowp
    ${TENSORFLOW_LITE_MICRO_DIR}/third_party_static/flatbuffers/include
    ${TENSORFLOW_LITE_MICRO_DIR}/third_party_static/ruy
  )

  zephyr_library_sources(
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/simple_memory_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/debug_log.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_error_reporter.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/all_ops_resolver.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/memory_helpers.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/test_helpers.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/recording_micro_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_time.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/recording_simple_memory_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_string.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_profiler.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/flatbuffer_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_graph.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/mock_micro_graph.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_interpreter.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/micro_allocator.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/system_setup.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/memory_planner/linear_memory_planner.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/c/common.c
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/kernels/internal/quantization_util.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/kernels/kernel_util.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/flatbuffer_conversions.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/error_reporter.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/tensor_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/core/api/op_resolver.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/schema/schema_utils.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/activations.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/activations_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/add.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/add_n.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/arg_min_max.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/batch_to_space_nd.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cast.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/ceil.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/circular_buffer.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/comparisons.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/concatenation.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/conv.cc # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/conv_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cumsum.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/depth_to_space.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/depthwise_conv.cc # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/depthwise_conv_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/dequantize.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/detection_postprocess.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/elementwise.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/elu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/ethosu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/exp.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/expand_dims.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/fill.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/floor.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/floor_div.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/floor_mod.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/fully_connected.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/fully_connected_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/gather.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/gather_nd.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/hard_swish.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/hard_swish_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/if.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/kernel_runner.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/kernel_util.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/l2norm.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/l2_pool_2d.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/leaky_relu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logical.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logical_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logistic.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/logistic_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/log_softmax.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/maximum_minimum.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/mul.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/neg.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/pack.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/pad.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/pooling.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/pooling_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/prelu.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/quantize.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/quantize_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/reduce.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/reshape.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/resize_bilinear.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/round.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/shape.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/softmax.cc  # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/softmax_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/space_to_batch_nd.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/space_to_depth.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/split.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/split_v.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/squeeze.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/strided_slice.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/sub.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/cmsis_nn/svdf.cc # use cmsis kernel
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/svdf_common.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/tanh.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/transpose.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/transpose_conv.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/unpack.cc
    ${TENSORFLOW_LITE_MICRO_DIR}/tensorflow/lite/micro/kernels/zeros_like.cc
  )

endif()
geekbozu commented 2 years ago

It builds now. My model just seems to be made with to new of a version of tensorflow as it just runs into an exit condition on invoke... Thanks for the help! Its a start :D

stephanosio commented 2 years ago

FYI @JordanYates

Yaxit commented 9 months ago

Should have been closed with the new module structure https://github.com/zephyrproject-rtos/zephyr/pull/56957