diff --git a/include/tim/vx/ops.h b/include/tim/vx/ops.h index f09d00f..52d57a2 100644 --- a/include/tim/vx/ops.h +++ b/include/tim/vx/ops.h @@ -99,5 +99,6 @@ #include "tim/vx/ops/topk.h" #include "tim/vx/ops/bidirectional_sequence_lstm.h" #include "tim/vx/ops/hashtable_lookup.h" +#include "tim/vx/ops/embedding_lookup.h" #endif /* TIM_VX_OPS_H_ */ diff --git a/include/tim/vx/ops/embedding_lookup.h b/include/tim/vx/ops/embedding_lookup.h new file mode 100644 index 0000000..c425a19 --- /dev/null +++ b/include/tim/vx/ops/embedding_lookup.h @@ -0,0 +1,49 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#ifndef TIM_VX_OPS_EMBEDDING_LOOKUP_H_ +#define TIM_VX_OPS_EMBEDDING_LOOKUP_H_ +#include "tim/vx/builtin_op.h" + +namespace tim { +namespace vx { +namespace ops { + +/** + * ## EmbeddingLookup + * + * Looks up sub-tensors in the input tensor with specific indices(idx) + */ + +class EmbeddingLookup : public BuiltinOp { + public: + EmbeddingLookup(Graph* Graph); + + std::shared_ptr Clone(std::shared_ptr& graph) const override; +}; + +} // namespace ops +} // namespace vx +} // namespace tim + +#endif /* TIM_VX_OPS_EMBEDDING_LOOKUP_H_ */ diff --git a/src/tim/vx/ops/README.md b/src/tim/vx/ops/README.md index 8bf63dd..a9c1268 100644 --- a/src/tim/vx/ops/README.md +++ b/src/tim/vx/ops/README.md @@ -128,8 +128,7 @@ MaxPool3d|MAX_POOL3D|Mapped|[Onnx.MaxPool](https://github.com/onnx/onnx/blob/mai |LSTMCell|LSTMUNIT_OVXLIB|replace with UnidirectionalSequenceLSTM by set n_step = 1 |[ANEURALNETWORKS_LSTM](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0ad0377e8c305e596fb7f64ff896671fc5) ||PRE_PROCESS|TBD |Image Preprocessing (YUV2RGB, Input Normalization, Resizing, etc) |HashtableLookup|HASHTABLE_LOOKUP|Mapped|[ANEURALNETWORKS_HASHTABLE_LOOKUP](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0aca92716c8c73c1f0fa7f0757916fee26) -||EMBEDDING_LOOKUP|Planned 22Q4|[ANEURALNETWORKS_EMBEDDING_LOOKUP](developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a8d2ada77adb74357fc0770405bca0e3) -||LSH_PROJECTION|Planned 22Q4|[ANEURALNETWORKS_LSH_PROJECTION](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a800cdcec5d7ba776789cb2d1ef669965) +|EmbeddingLookup|EMBEDDING_LOOKUP|Mapped|[ANEURALNETWORKS_EMBEDDING_LOOKUP](developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a8d2ada77adb74357fc0770405bca0e3) ||HEATMAP_MAX_KEYPOINT|TBD|[ANEURALNETWORKS_HEATMAP_MAX_KEYPOINT](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a5ffccf92d127766a741225ff7ad6f743) ||AXIS_ALIGNED_BBOX_TRANSFORM|TBD|[ANEURALNETWORKS_AXIS_ALIGNED_BBOX_TRANSFORM](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0afd7603dd54060e6a52f5861674448528) ||BOX_WITH_NMS_LIMIT|TBD|[ANEURALNETWORKS_BOX_WITH_NMX_LIMIT](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a2d81e878c19e15700dad111ba6c0be89) @@ -145,6 +144,7 @@ MaxPool3d|MAX_POOL3D|Mapped|[Onnx.MaxPool](https://github.com/onnx/onnx/blob/mai OneHot|ONE_HOT|Mapped|[tf.one_hot](https://tensorflow.google.cn/api_docs/python/tf/one_hot) ||NMS| TBD |[tf.image.non_max_suppression](https://tensorflow.google.cn/api_docs/python/tf/image/non_max_suppression) ||SCATTER_ND_UPDATE|Planned 21Q4|[tf.compat.v1.scatter_nd_update](https://tensorflow.google.cn/api_docs/python/tf/compat/v1/scatter_nd_update) +||LSH_PROJECTION|Deprecated ||CONV_RELU|Deprecated ||CONV_RELU_POOL|Deprecated ||FCL|Deprecated diff --git a/src/tim/vx/ops/embedding_lookup.cc b/src/tim/vx/ops/embedding_lookup.cc new file mode 100644 index 0000000..f5a1910 --- /dev/null +++ b/src/tim/vx/ops/embedding_lookup.cc @@ -0,0 +1,41 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#include "tim/vx/ops/embedding_lookup.h" + +#include "builtin_op_impl.h" +#include "vsi_nn_pub.h" + +namespace tim { +namespace vx { +namespace ops { + +EmbeddingLookup::EmbeddingLookup(Graph* graph) : BuiltinOp(graph, VSI_NN_OP_EMBEDDING_LOOKUP,2,1) {} + +std::shared_ptr EmbeddingLookup::Clone(std::shared_ptr& graph) const { + return graph->CreateOperation(); +} + +} // namespace ops +} // namespace vx +} // namespace tim \ No newline at end of file diff --git a/src/tim/vx/ops/embedding_lookup_test.cc b/src/tim/vx/ops/embedding_lookup_test.cc new file mode 100644 index 0000000..4683dc7 --- /dev/null +++ b/src/tim/vx/ops/embedding_lookup_test.cc @@ -0,0 +1,66 @@ +/**************************************************************************** +* +* Copyright (c) 2022 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#include "tim/vx/context.h" +#include "tim/vx/graph.h" +#include "tim/vx/ops/embedding_lookup.h" + +#include "gtest/gtest.h" + +TEST(EmbeddingLookup, shape_2_5_int32) { + auto ctx = tim::vx::Context::Create(); + auto graph = ctx->CreateGraph(); + tim::vx::ShapeType idx_shape({3}); + tim::vx::ShapeType lut_shape({2, 5}); + tim::vx::ShapeType out_shape({2, 3}); + + tim::vx::TensorSpec idx_spec(tim::vx::DataType::INT32, idx_shape, + tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec lut_spec(tim::vx::DataType::INT32, lut_shape, + tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec out_spec(tim::vx::DataType::INT32, out_shape, + tim::vx::TensorAttribute::OUTPUT); + auto idx_tensor = graph->CreateTensor(idx_spec); + auto lut_tensor = graph->CreateTensor(lut_spec); + auto out_tensor = graph->CreateTensor(out_spec); + + std::vector idx_data = {0,3,4}; + std::vector lut_data = { + 1,2,3,4,5,6,7,8,9,10}; + std::vector golden = { + 1,2,7,8,9,10}; + + EXPECT_TRUE(idx_tensor->CopyDataToTensor(idx_data.data(), + idx_data.size() * sizeof(int32_t))); + EXPECT_TRUE(lut_tensor->CopyDataToTensor(lut_data.data(), + lut_data.size() * sizeof(int32_t))); + auto op = graph->CreateOperation(); + (*op).BindInputs({idx_tensor,lut_tensor}).BindOutput(out_tensor); + + EXPECT_TRUE(graph->Compile()); + EXPECT_TRUE(graph->Run()); + + std::vector output(golden.size()); + EXPECT_TRUE(out_tensor->CopyDataFromTensor(output.data())); + EXPECT_EQ(golden, output); +} \ No newline at end of file