diff --git a/README.md b/README.md index ce06f43..dca1c77 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ All install files (both headers and *.so) is located in : `host_build/install` cmake options: | option name | Summary | Default | -| ----- | ----- | ----- | +| ----- | ----- | ----- | |`TIM_VX_ENABLE_TEST`| Enable unit test case for public APIs and ops | OFF | |`TIM_VX_ENABLE_LAYOUT_INFER`| Build with tensor data layout inference support| ON | |`TIM_VX_USE_EXTERNAL_OVXLIB`| Replace internal with a prebuilt libovxlib library | OFF | @@ -87,7 +87,7 @@ Run unit test: cd host_build/src/tim export LD_LIBRARY_PATH=`pwd`/../../../prebuilt-sdk/x86_64_linux/lib::$LD_LIBRARY_PATH -export VIVANTE_SDK_DIR=`pwd`/../../../prebuilt-sdk/x86_64_linux/lib +export VIVANTE_SDK_DIR=`pwd`/../../../prebuilt-sdk/x86_64_linux/ export VSIMULATOR_CONFIG= # if you want to debug wit gdb, please set export DISABLE_IDE_DEBUG=1 @@ -109,7 +109,7 @@ export DISABLE_IDE_DEBUG=1 1. prepare toolchain file follow cmake standard 2. make sure cross build low-level driver with toolchain separately, we need the sdk from the low-level driver 3. add ```-DEXTERNAL_VIV_SDK=``` to cmake definitions, also remember ```-DCMAKE_TOOLCHAIN_FILE=``` -4. or for using a buildroot toolchain with extrnal VIV-SDK add: +4. or for using a buildroot toolchain with extrnal VIV-SDK add: ```cmake -DCONFIG=BUILDROOT -DCMAKE_SYSROOT=${CMAKE_SYSROOT} -DEXTERNAL_VIV_SDK=${BUILDROOT_SYSROOT} ``` diff --git a/include/tim/vx/ops/roi_align.h b/include/tim/vx/ops/roi_align.h new file mode 100644 index 0000000..dead1b6 --- /dev/null +++ b/include/tim/vx/ops/roi_align.h @@ -0,0 +1,72 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#ifndef TIM_VX_OPS_ROI_ALIGN_H_ +#define TIM_VX_OPS_ROI_ALIGN_H_ +#include "tim/vx/direct_map_op.h" + +namespace tim { +namespace vx { +namespace ops { + +/** + * ## ROI_ALIGN + * + * Select and scale the feature map of each region of interest to a unified output + * size by average pooling sampling points from bilinear interpolation. + * + * - output_height : specifying the output height of the output tensor. + * - output_width : specifying the output width of the output tensor. + * - height_ratio : specifying the ratio from the height of original image to the + * height of feature map. + * - width_ratio : specifying the ratio from the width of original image to the + * width of feature map. + * - height_sample_num : specifying the number of sampling points in height dimension + * used to compute the output. + * - width_sample_num :specifying the number of sampling points in width dimension + * used to compute the output. + */ + +class ROI_Align : public DirectMapOp { + public: + ROI_Align(Graph* graph, int32_t output_height, int32_t output_width, + float height_ratio, float width_ratio, int32_t height_sample_num, + int32_t width_sample_num); + + std::shared_ptr Clone( + std::shared_ptr& graph) const override; + + protected: + int32_t output_height_; + int32_t output_width_; + float height_ratio_; + float width_ratio_; + int32_t height_sample_num_; + int32_t width_sample_num_; +}; + +} // namespace ops +} // namespace vx +} // namespace tim + +#endif /* TIM_VX_OPS_ROI_ALIGN_H_ */ diff --git a/src/tim/vx/ops/README.md b/src/tim/vx/ops/README.md index 43ee745..73aa1b4 100644 --- a/src/tim/vx/ops/README.md +++ b/src/tim/vx/ops/README.md @@ -109,8 +109,8 @@ GroupedConv1d|GROUPED_CONV1D|Mapped|[tf.keras.layers.Conv1D](https://tensorflow. |BroadCast|EXPAND_BROADCAST|Mapped|[numpy.broadcast_to](https://numpy.org/doc/stable/reference/generated/numpy.broadcast_to.html) ||PROPOSAL| TBD |[Faster-RCNN Proposal Layer](https://github.com/intel/caffe/blob/master/examples/faster-rcnn/lib/rpn/proposal_layer.py) ||ROI_POOL|Planned 22Q4|[ANEURALNETWORKS_ROI_POOLING](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a6736198af337b2efbdb0b6b64dee7fe4) -||ROI_ALIGN|Planned 22Q2|[ANEURALNETWORKS_ROI_ALIGN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a2848b39dd4bfba78f2438fda0d9397a4) -||TOPK|Planned 22Q2 (limited support)|[tf.math.top_k](https://tensorflow.google.cn/api_docs/python/tf/math/top_k) +ROI_Align||ROI_ALIGN|Mapped|[ANEURALNETWORKS_ROI_ALIGN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a2848b39dd4bfba78f2438fda0d9397a4) +TopK||TOPK|Mapped (limited support)|[tf.math.top_k](https://tensorflow.google.cn/api_docs/python/tf/math/top_k) |GRUCell|GRUCELL_OVXLIB|Planned 22Q3|[tf.keras.layers.GRUCell](https://tensorflow.google.cn/api_docs/python/tf/keras/layers/GRUCell?hl=en) |UnidirectionalSequenceGRU|GRU_OVXLIB|Planned 22Q3|[tf.keras.layers.GRU](https://tensorflow.google.cn/api_docs/python/tf/keras/layers/GRUCell?hl=en) |UnidirectionalSequenceRNN|UNIDIRECTIONAL_SEQUENCE_RNN|Planned 22Q3|[ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_RNN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0ae11aa1d461d2abaa117f6ee2cb503dd8) diff --git a/src/tim/vx/ops/conv2d_test.cc b/src/tim/vx/ops/conv2d_test.cc index 08b709d..cb24499 100644 --- a/src/tim/vx/ops/conv2d_test.cc +++ b/src/tim/vx/ops/conv2d_test.cc @@ -1,3 +1,26 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ #include "tim/vx/ops/conv2d.h" #include "gtest/gtest.h" diff --git a/src/tim/vx/ops/roi_align.cc b/src/tim/vx/ops/roi_align.cc new file mode 100644 index 0000000..d9ad4d1 --- /dev/null +++ b/src/tim/vx/ops/roi_align.cc @@ -0,0 +1,61 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#include "tim/vx/ops/roi_align.h" + +#include "direct_map_op_impl.h" +#include "vsi_nn_pub.h" + +namespace tim { +namespace vx { +namespace ops { + +ROI_Align::ROI_Align(Graph* graph, int32_t output_height, int32_t output_width, + float height_ratio, float width_ratio, int32_t height_sample_num, + int32_t width_sample_num) + : DirectMapOp(graph, VSI_NN_OP_ROI_ALIGN), + output_height_(output_height), + output_width_(output_width), + height_ratio_(height_ratio), + width_ratio_(width_ratio), + height_sample_num_(height_sample_num), + width_sample_num_(width_sample_num) { + this->impl()->node()->nn_param.roi_align.output_height = output_height; + this->impl()->node()->nn_param.roi_align.output_width = output_width; + this->impl()->node()->nn_param.roi_align.height_ratio = height_ratio; + this->impl()->node()->nn_param.roi_align.width_ratio = width_ratio; + this->impl()->node()->nn_param.roi_align.height_sample_num = + height_sample_num; + this->impl()->node()->nn_param.roi_align.width_sample_num = width_sample_num; +} + +std::shared_ptr ROI_Align::Clone( + std::shared_ptr& graph) const { + return graph->CreateOperation( + this->output_height_, this->output_width_, this->height_ratio_, + this->width_ratio_, this->height_sample_num_, this->width_sample_num_); +} + +} // namespace ops +} // namespace vx +} // namespace tim \ No newline at end of file diff --git a/src/tim/vx/ops/roi_align_test.cc b/src/tim/vx/ops/roi_align_test.cc new file mode 100644 index 0000000..1785e13 --- /dev/null +++ b/src/tim/vx/ops/roi_align_test.cc @@ -0,0 +1,106 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#include "tim/vx/ops/roi_align.h" + +#include "gtest/gtest.h" +#include "test_utils.h" +#include "tim/vx/context.h" +#include "tim/vx/graph.h" +#include "tim/vx/types.h" + +TEST(ROI_Align, shape_4_2_1_1_float32) { + auto ctx = tim::vx::Context::Create(); + auto graph = ctx->CreateGraph(); + + uint32_t height = 4; + uint32_t width = 4; + uint32_t channels = 1; + uint32_t batch = 1; + uint32_t num_rois = 4; + uint32_t depth = channels; + + int32_t out_height = 2; + int32_t out_width = 2; + float height_ratio = 2.0f; + float width_ratio = 2.0f; + int32_t height_sample_num = 4; + int32_t width_sample_num = 4; + + tim::vx::ShapeType input_shape({width, height, channels, batch}); //whcn + tim::vx::ShapeType regions_shape({num_rois, 4}); + tim::vx::ShapeType batch_index_shape({num_rois}); + tim::vx::ShapeType output_shape( + {(uint32_t)out_width, (uint32_t)out_height, depth, num_rois}); + + tim::vx::TensorSpec input_spec(tim::vx::DataType::FLOAT32, input_shape, + tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec regions_spec(tim::vx::DataType::FLOAT32, regions_shape, + tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec batch_index_spec(tim::vx::DataType::INT32, + batch_index_shape, + tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec output_spec(tim::vx::DataType::FLOAT32, output_shape, + tim::vx::TensorAttribute::OUTPUT); + + std::vector input_data = {-10.0f, -1.0f, 4.0f, -5.0f, -8.0f, -2.0f, + 9.0f, 1.0f, 7.0f, -2.0f, 3.0f, -7.0f, + -2.0f, 10.0f, -3.0f, 5.0f}; + + std::vector regions_data = {2.0f, 2.0f, 4.0f, 4.0f, 0.0f, 0.0f, + 8.0f, 8.0f, 2.0f, 0.0f, 4.0f, 8.0f, + 0.0f, 2.0f, 8.0f, 4.0f}; + + std::vector batch_index_data = {0, 0, 0, 0}; + + std::vector golden = { + 0.375f, 5.125f, -0.375f, 2.875f, -0.5f, -0.3125f, 3.1875f, 1.125f, + 0.25f, 4.25f, 4.875f, 0.625f, -0.1875f, 1.125f, 0.9375f, -2.625f}; + + auto input_tensor = graph->CreateTensor(input_spec); + auto regions_tensor = graph->CreateTensor(regions_spec, regions_data.data()); + auto batch_index_tensor = + graph->CreateTensor(batch_index_spec, batch_index_data.data()); + auto output_tensor = graph->CreateTensor(output_spec); + + auto roi_align = graph->CreateOperation( + out_height, out_width, height_ratio, width_ratio, height_sample_num, + width_sample_num); + (*roi_align) + .BindInput(input_tensor) + .BindInput(regions_tensor) + .BindInput(batch_index_tensor) + .BindOutput(output_tensor); + + EXPECT_TRUE(graph->Compile()); + + input_tensor->CopyDataToTensor(input_data.data()); + regions_tensor->CopyDataToTensor(regions_data.data()); + batch_index_tensor->CopyDataToTensor(batch_index_data.data()); + + EXPECT_TRUE(graph->Run()); + + std::vector output(num_rois * out_height * out_width * depth); + EXPECT_TRUE(output_tensor->CopyDataFromTensor(output.data())); + EXPECT_EQ(golden, output); +} \ No newline at end of file