diff --git a/src/tim/transform/layout_inference.cc b/src/tim/transform/layout_inference.cc index 7f5eb82..4ad680f 100644 --- a/src/tim/transform/layout_inference.cc +++ b/src/tim/transform/layout_inference.cc @@ -64,6 +64,7 @@ #include "ops/transpose_layout_inference.h" #include "ops/unidirectional_lstm_layout_inference.h" #include "ops/broadcast_layout_inference.h" +#include "ops/unidirectional_rnn_layout_inference.h" #include #include @@ -267,6 +268,7 @@ std::vector> HandleLayoutInfer( REGIST_LAYOUT_INFERENCE(VSI_NN_OP_CONV3D, Conv3d); REGIST_LAYOUT_INFERENCE(VSI_NN_OP_LSTM_OVXLIB, UnidirectionalLstm); REGIST_LAYOUT_INFERENCE(VSI_NN_OP_EXPAND_BROADCAST, Broadcast); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_UNIDIRECTIONAL_SEQUENCE_RNN, UnidirectionalRnn); REGIST_LOGICAL_LAYOUT_INFERENCE(VSI_NN_OP_LOGICAL_OPS); REGIST_REDUCE_LAYOUT_INFERENCE(VSI_NN_OP_REDUCE); // use default layout inference diff --git a/src/tim/transform/ops/unidirectional_rnn_layout_inference.h b/src/tim/transform/ops/unidirectional_rnn_layout_inference.h new file mode 100644 index 0000000..f88dbdc --- /dev/null +++ b/src/tim/transform/ops/unidirectional_rnn_layout_inference.h @@ -0,0 +1,99 @@ +/**************************************************************************** + * + * Copyright (c) 2022 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_UNIDIRECTIONAL_RNN_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_UNIDIRECTIONAL_RNN_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/reshape.h" +#include "tim/vx/ops/nbg.h" +#include "tim/vx/ops/transpose.h" +#include "tim/vx/ops/batchnorm.h" +#include "tim/vx/ops/clip.h" + +#include "ops/op_layout_inference.h" +#include "permute_vector.h" +#include "builtin_op_impl.h" + +namespace tim { +namespace transform { + +class UnidirectionalRnnLayoutInfer : public OpLayoutInfer { + public: + UnidirectionalRnnLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + // reverse any applied permute on it's input tensor + void OnInputs( + std::vector>& next_tensors) override { + ReverseInputsPermuteVector(); + + auto cloned_op = op_->Clone(context_->infer_graph_); + + for (const auto& i_src : op_->impl()->InputsTensor()) { + std::shared_ptr infer_tensor; + std::shared_ptr required_pv; + if ((i_src->IsConstTensor() && + !(i_src->GetSpec().attr_ & vx::TensorAttribute::INPUT))) { + infer_tensor = context_->infer_graph_->CreateTensor( + i_src->GetSpec(), i_src->GetDataRef()); + context_->UpdateTensorMap(i_src, infer_tensor); + } + if (i_src->GetId() == (uint32_t)-1) { + infer_tensor = context_->infer_graph_->CreateTensorPlaceHolder(); + context_->UpdateTensorMap(i_src, infer_tensor); + } + required_pv = MakeShared(i_src->GetShape().size()); + context_->SetPermuteVector(i_src, required_pv); + } + + + for (const auto& i_src : op_->impl()->InputsTensor()) { + (*cloned_op).BindInput(context_->GetMapedTensor(i_src)); + } + + + std::vector> required_pv_lst; + for (auto out_tensor : op_->impl()->OutputsTensor()) { + std::shared_ptr infer_tensor; + if (out_tensor->GetId() == (uint32_t)-1) { + out_tensor = context_->infer_graph_->CreateTensorPlaceHolder(); + } + required_pv_lst.push_back(MakeShared(out_tensor->GetShape().size())); + } + auto out_infer = CreateOutputsTensor(required_pv_lst); + + (*cloned_op).BindOutputs(out_infer); + uint32_t i = 0; + for (auto out_tensor : op_->impl()->OutputsTensor()) { + context_->SetPermuteVector(out_tensor, required_pv_lst[i++]); + next_tensors.push_back(out_tensor); + } + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/vx/ops/unidirectional_sequence_rnn.cc b/src/tim/vx/ops/unidirectional_sequence_rnn.cc index fd26ce1..8234971 100644 --- a/src/tim/vx/ops/unidirectional_sequence_rnn.cc +++ b/src/tim/vx/ops/unidirectional_sequence_rnn.cc @@ -1,6 +1,6 @@ /**************************************************************************** * -* Copyright (c) 2020 Vivante Corporation +* Copyright (c) 2022 Vivante Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), diff --git a/src/tim/vx/ops/unidirectional_sequence_rnn_test.cc b/src/tim/vx/ops/unidirectional_sequence_rnn_test.cc index f18e2f7..07746c7 100644 --- a/src/tim/vx/ops/unidirectional_sequence_rnn_test.cc +++ b/src/tim/vx/ops/unidirectional_sequence_rnn_test.cc @@ -1,6 +1,6 @@ /**************************************************************************** * -* Copyright (c) 2021 Vivante Corporation +* Copyright (c) 2022 Vivante Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -28,19 +28,19 @@ #include "test_utils.h" #include "gtest/gtest.h" -TEST(UnidirectionalSequenceRnn, shape_2_3_4_float_sigmoid) { +TEST(UnidirectionalSequenceRnn, shape_2_3_2_float_sigmoid) { auto ctx = tim::vx::Context::Create(); auto graph = ctx->CreateGraph(); - uint32_t input_size = 2, batch_size = 3, num_units = 4; + uint32_t input_size = 2, batch_size = 3, num_units = 4, time_step = 2; - tim::vx::ShapeType input_shape({input_size, batch_size, 2}); + tim::vx::ShapeType input_shape({input_size, batch_size, time_step}); tim::vx::ShapeType weights_shape({input_size, num_units}); tim::vx::ShapeType recurrent_weights_shape({num_units, num_units}); tim::vx::ShapeType bias_shape({num_units}); tim::vx::ShapeType recurrent_bias_shape({num_units}); tim::vx::ShapeType state_in_shape({num_units, batch_size}); - tim::vx::ShapeType output_shape({num_units, batch_size, 2}); + tim::vx::ShapeType output_shape({num_units, batch_size, time_step}); tim::vx::ShapeType state_out_shape({num_units, batch_size}); tim::vx::TensorSpec input_spec(tim::vx::DataType::FLOAT32, @@ -144,19 +144,19 @@ TEST(UnidirectionalSequenceRnn, shape_2_3_4_float_sigmoid) { } -TEST(UnidirectionalSequenceRnn, shape_2_3_4_float_relu) { +TEST(UnidirectionalSequenceRnn, shape_2_3_2_float_relu) { auto ctx = tim::vx::Context::Create(); auto graph = ctx->CreateGraph(); - uint32_t input_size = 2, batch_size = 3, num_units = 4; + uint32_t input_size = 2, batch_size = 3, num_units = 4, time_step = 2; - tim::vx::ShapeType input_shape({input_size, batch_size, 2}); + tim::vx::ShapeType input_shape({input_size, batch_size, time_step}); tim::vx::ShapeType weights_shape({input_size, num_units}); tim::vx::ShapeType recurrent_weights_shape({num_units, num_units}); tim::vx::ShapeType bias_shape({num_units}); tim::vx::ShapeType recurrent_bias_shape({num_units}); tim::vx::ShapeType state_in_shape({num_units, batch_size}); - tim::vx::ShapeType output_shape({num_units, batch_size, 2}); + tim::vx::ShapeType output_shape({num_units, batch_size, time_step}); tim::vx::ShapeType state_out_shape({num_units, batch_size}); tim::vx::TensorSpec input_spec(tim::vx::DataType::FLOAT32,