From 748274143bdc2a94484042ff6498750638776e3b Mon Sep 17 00:00:00 2001 From: liyuenan <37231553+liyuenan2333@users.noreply.github.com> Date: Thu, 13 May 2021 22:27:23 +0800 Subject: [PATCH] support layout inference for operations (#39) Add layout inference support for space2depth, depth2space, space2batch, batch2space, pad and reduce. Signed-off-by: yuenan.li Co-authored-by: yuenan.li --- include/tim/vx/ops/batch2space.h | 7 +- include/tim/vx/ops/depth2space.h | 3 +- include/tim/vx/ops/reduce.h | 1 + include/tim/vx/ops/space2batch.h | 7 +- include/tim/vx/ops/space2depth.h | 3 +- src/tim/transform/layout_inference.cc | 29 +++++++ .../ops/batch2space_layout_inference.h | 84 +++++++++++++++++++ .../ops/depth2space_layout_inference.h | 77 +++++++++++++++++ src/tim/transform/ops/op_layout_inference.cc | 12 +++ src/tim/transform/ops/op_layout_inference.h | 3 + src/tim/transform/ops/pad_layout_inference.h | 74 ++++++++++++++++ .../transform/ops/reduce_layout_inference.h | 4 +- .../ops/space2batch_layout_inference.h | 84 +++++++++++++++++++ .../ops/space2depth_layout_inference.h | 78 +++++++++++++++++ src/tim/vx/ops/batch2space.cc | 6 +- src/tim/vx/ops/depth2space.cc | 5 +- src/tim/vx/ops/reduce.cc | 1 + src/tim/vx/ops/space2batch.cc | 6 +- src/tim/vx/ops/space2depth.cc | 6 +- 19 files changed, 471 insertions(+), 19 deletions(-) create mode 100644 src/tim/transform/ops/batch2space_layout_inference.h create mode 100644 src/tim/transform/ops/depth2space_layout_inference.h create mode 100644 src/tim/transform/ops/pad_layout_inference.h create mode 100644 src/tim/transform/ops/space2batch_layout_inference.h create mode 100644 src/tim/transform/ops/space2depth_layout_inference.h diff --git a/include/tim/vx/ops/batch2space.h b/include/tim/vx/ops/batch2space.h index be4211e..cd9257c 100644 --- a/include/tim/vx/ops/batch2space.h +++ b/include/tim/vx/ops/batch2space.h @@ -32,10 +32,11 @@ namespace tim { namespace vx { namespace ops { -class Batch2Space : public Operation { +class BatchToSpace : public Operation { public: - Batch2Space(Graph* graph, const std::vector& block_size, - const std::vector& crop); + BatchToSpace(Graph* graph, const std::vector& block_size, + const std::vector& crop, + DataLayout layout = DataLayout::WHCN); protected: std::vector block_size_; diff --git a/include/tim/vx/ops/depth2space.h b/include/tim/vx/ops/depth2space.h index eef90e6..be2a6d1 100644 --- a/include/tim/vx/ops/depth2space.h +++ b/include/tim/vx/ops/depth2space.h @@ -31,7 +31,8 @@ namespace ops { class DepthToSpace : public Operation { public: - DepthToSpace(Graph* Graph, int block_size); + DepthToSpace(Graph* Graph, int block_size, + DataLayout layout = DataLayout::WHCN); protected: int block_size_; diff --git a/include/tim/vx/ops/reduce.h b/include/tim/vx/ops/reduce.h index 1c7f381..5eafd92 100644 --- a/include/tim/vx/ops/reduce.h +++ b/include/tim/vx/ops/reduce.h @@ -45,6 +45,7 @@ DECLARE_REDUCE_OP(Max); DECLARE_REDUCE_OP(Any); DECLARE_REDUCE_OP(Prod); DECLARE_REDUCE_OP(Mean); +DECLARE_REDUCE_OP(Sum); #undef DECLARE_REDUCE_OP diff --git a/include/tim/vx/ops/space2batch.h b/include/tim/vx/ops/space2batch.h index 81e2f9d..484c9b4 100644 --- a/include/tim/vx/ops/space2batch.h +++ b/include/tim/vx/ops/space2batch.h @@ -32,10 +32,11 @@ namespace tim { namespace vx { namespace ops { -class Space2Batch : public Operation { +class SpaceToBatch : public Operation { public: - Space2Batch(Graph* graph, const std::vector& block_size, - const std::vector& pad); + SpaceToBatch(Graph* graph, const std::vector& block_size, + const std::vector& pad, + DataLayout layout = DataLayout::WHCN); protected: std::vector block_size_; diff --git a/include/tim/vx/ops/space2depth.h b/include/tim/vx/ops/space2depth.h index fe5b4b7..5acd6b4 100644 --- a/include/tim/vx/ops/space2depth.h +++ b/include/tim/vx/ops/space2depth.h @@ -31,7 +31,8 @@ namespace ops { class SpaceToDepth : public Operation { public: - SpaceToDepth(Graph* graph, std::vector block_size); + SpaceToDepth(Graph* graph, std::vector block_size, + DataLayout layout = DataLayout::WHCN); protected: std::vector block_size_; diff --git a/src/tim/transform/layout_inference.cc b/src/tim/transform/layout_inference.cc index 3d17bd8..7bfc135 100644 --- a/src/tim/transform/layout_inference.cc +++ b/src/tim/transform/layout_inference.cc @@ -36,6 +36,12 @@ #include "ops/softmax_layout_inference.h" #include "ops/squeeze_layout_inference.h" #include "ops/stack_layout_inference.h" +#include "ops/space2depth_layout_inference.h" +#include "ops/depth2space_layout_inference.h" +#include "ops/space2batch_layout_inference.h" +#include "ops/batch2space_layout_inference.h" +#include "ops/pad_layout_inference.h" +#include "ops/reduce_layout_inference.h" #include #include @@ -127,6 +133,23 @@ void LayoutInferContext::UpdateGraphInputMap(const std::shared_ptr& break; \ } \ +#define REGIST_REDUCE_LAYOUT_INFERENCE(op_idx) \ + case op_idx: { \ + auto reduce_type = op->impl()->node()->nn_param.reduce.type; \ + switch (reduce_type) { \ + REGIST_LAYOUT_INFERENCE(VSI_NN_REDUCE_MEAN, ReduceMean); \ + REGIST_LAYOUT_INFERENCE(VSI_NN_REDUCE_MAX, ReduceMax); \ + REGIST_LAYOUT_INFERENCE(VSI_NN_REDUCE_MIN, ReduceMin); \ + REGIST_LAYOUT_INFERENCE(VSI_NN_REDUCE_PROD, ReduceProd); \ + REGIST_LAYOUT_INFERENCE(VSI_NN_REDUCE_ANY, ReduceAny); \ + REGIST_LAYOUT_INFERENCE(VSI_NN_REDUCE_SUM, ReduceSum); \ + default: \ + VSILOGW("Op %d: Default layout inference pass for reduce.", reduce_type);\ + assert(false); \ + } \ + break; \ + } \ + std::vector> HandleLayoutInfer( std::shared_ptr& ctx, const std::shared_ptr& op) { @@ -169,6 +192,12 @@ std::vector> HandleLayoutInfer( REGIST_LAYOUT_INFERENCE(VSI_NN_OP_SOFTMAX, Softmax); REGIST_LAYOUT_INFERENCE(VSI_NN_OP_SQUEEZE, Squeeze); REGIST_LAYOUT_INFERENCE(VSI_NN_OP_STACK, Stack); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_SPACE2DEPTH, SpaceToDepth); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_DEPTH2SPACE, DepthToSpace); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_SPACE2BATCH, SpaceToBatch); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_BATCH2SPACE, BatchToSpace); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_PAD, Pad); + REGIST_REDUCE_LAYOUT_INFERENCE(VSI_NN_OP_REDUCE); default: VSILOGW("Op %d: Default layout inference pass.", op_id); diff --git a/src/tim/transform/ops/batch2space_layout_inference.h b/src/tim/transform/ops/batch2space_layout_inference.h new file mode 100644 index 0000000..65fab8d --- /dev/null +++ b/src/tim/transform/ops/batch2space_layout_inference.h @@ -0,0 +1,84 @@ +/**************************************************************************** + * + * Copyright (c) 2020 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_BATCH2SPACE_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_BATCH2SPACE_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/batch2space.h" + +#include "src/tim/transform/ops/op_layout_inference.h" +#include "src/tim/transform/permute_vector.h" +#include "src/tim/vx/operation_private.h" +namespace tim { +namespace transform { +class BatchToSpaceLayoutInfer : public OpLayoutInfer { + public: + BatchToSpaceLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + void OnInputs( + std::vector>& next_tensors) override { + vx::DataLayout layout = op_->impl()->layout_; + auto required_pv = MakeShared(4); + if (layout == vx::DataLayout::CWHN) { + required_pv = std::make_shared>(kCWHN2WHCN); + } + auto input_tensors = op_->impl()->InputsTensor(); + + auto pv = context_->GetPermuteVector(input_tensors[0]); + auto final_pv = pv->Reverse()->Add(required_pv); + if (!final_pv->IsAligned()) { + auto perm_out = + InsertPermute(context_->GetMapedTensor(input_tensors[0]), final_pv); + context_->UpdateTensorMap(input_tensors[0], perm_out); + context_->SetPermuteVector(input_tensors[0], required_pv); + } + + uint32_t block_size_num = + op_->impl()->node()->nn_param.batch2space.block_size_num; + std::vector block_size(block_size_num); + memcpy(block_size.data(), + op_->impl()->node()->nn_param.batch2space.block_size, + sizeof(int) * block_size_num); + std::vector crop(4); + memcpy(crop.data(), op_->impl()->node()->nn_param.batch2space.crop, + sizeof(int) * 4); + + auto batch2space = + context_->infer_graph_->CreateOperation( + block_size, crop, vx::DataLayout::WHCN); + auto out_tensor_infer = CreateOutputsTensor(required_pv); + (*batch2space).BindInput(context_->GetMapedTensor(input_tensors[0])); + (*batch2space).BindOutput(out_tensor_infer[0]); + context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], required_pv); + // Add out tensor of src_graph into next_tensor + next_tensors.push_back(op_->impl()->OutputsTensor()[0]); + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/transform/ops/depth2space_layout_inference.h b/src/tim/transform/ops/depth2space_layout_inference.h new file mode 100644 index 0000000..7b92fb7 --- /dev/null +++ b/src/tim/transform/ops/depth2space_layout_inference.h @@ -0,0 +1,77 @@ +/**************************************************************************** + * + * Copyright (c) 2020 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_DEPTH2SPACE_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_DEPTH2SPACE_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/depth2space.h" + +#include "src/tim/transform/ops/op_layout_inference.h" +#include "src/tim/transform/permute_vector.h" +#include "src/tim/vx/operation_private.h" + +namespace tim { +namespace transform { +class DepthToSpaceLayoutInfer : public OpLayoutInfer { + public: + DepthToSpaceLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + void OnInputs( + std::vector>& next_tensors) override { + vx::DataLayout layout = op_->impl()->layout_; + auto required_pv = MakeShared(4); + if (layout == vx::DataLayout::CWHN) { + required_pv = std::make_shared>(kCWHN2WHCN); + } + auto input_tensors = op_->impl()->InputsTensor(); + + auto pv = context_->GetPermuteVector(input_tensors[0]); + auto final_pv = pv->Reverse()->Add(required_pv); + if (!final_pv->IsAligned()) { + auto perm_out = + InsertPermute(context_->GetMapedTensor(input_tensors[0]), final_pv); + context_->UpdateTensorMap(input_tensors[0], perm_out); + context_->SetPermuteVector(input_tensors[0], required_pv); + } + + int block_size = op_->impl()->node()->nn_param.depth2space.block_size; + + auto space2depth = + context_->infer_graph_->CreateOperation( + block_size, vx::DataLayout::WHCN); + auto out_tensor_infer = CreateOutputsTensor(required_pv); + (*space2depth).BindInput(context_->GetMapedTensor(input_tensors[0])); + (*space2depth).BindOutput(out_tensor_infer[0]); + context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], required_pv); + // Add out tensor of src_graph into next_tensor + next_tensors.push_back(op_->impl()->OutputsTensor()[0]); + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/transform/ops/op_layout_inference.cc b/src/tim/transform/ops/op_layout_inference.cc index cc5a755..716c407 100644 --- a/src/tim/transform/ops/op_layout_inference.cc +++ b/src/tim/transform/ops/op_layout_inference.cc @@ -234,5 +234,17 @@ std::shared_ptr OpLayoutInfer::PermuteConstTensor( } return context_->infer_graph_->CreateTensor(dst_spec, data.data()); } + +std::vector OpLayoutInfer::MapPadding(const std::vector& perm, + const std::vector& padding) { + assert(perm.size() == padding.size()); + std::vector r(padding.size()); + + for (int i = 0; i < padding.size(); ++i) { + r[i] = padding[perm[i]]; + } + + return r; +} } // namespace transform } // namespace tim \ No newline at end of file diff --git a/src/tim/transform/ops/op_layout_inference.h b/src/tim/transform/ops/op_layout_inference.h index e86e1a7..3badeee 100644 --- a/src/tim/transform/ops/op_layout_inference.h +++ b/src/tim/transform/ops/op_layout_inference.h @@ -75,6 +75,9 @@ class OpLayoutInfer { std::shared_ptr PermuteConstTensor( const std::shared_ptr& input, const std::shared_ptr& pv); + + std::vector MapPadding(const std::vector& perm, + const std::vector& padding); protected: const std::shared_ptr op_; diff --git a/src/tim/transform/ops/pad_layout_inference.h b/src/tim/transform/ops/pad_layout_inference.h new file mode 100644 index 0000000..4d62b26 --- /dev/null +++ b/src/tim/transform/ops/pad_layout_inference.h @@ -0,0 +1,74 @@ +/**************************************************************************** + * + * Copyright (c) 2020 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_PAD_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_PAD_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/pad.h" + +#include "src/tim/transform/ops/op_layout_inference.h" +#include "src/tim/transform/permute_vector.h" +#include "src/tim/vx/operation_private.h" +namespace tim { +namespace transform { +class PadLayoutInfer : public OpLayoutInfer { + public: + PadLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + void OnInputs( + std::vector>& next_tensors) override { + assert(op_->impl()->InputsTensor().size() == 1); + auto i_src = op_->impl()->InputsTensor()[0]; + auto input_pv = context_->GetPermuteVector(i_src); + + uint32_t dim_num = op_->impl()->node()->nn_param.pad.dim_num; + std::vector front_size(dim_num); + std::vector back_size(dim_num); + memcpy(front_size.data(), op_->impl()->node()->nn_param.pad.front_size, + sizeof(uint32_t) * dim_num); + memcpy(back_size.data(), op_->impl()->node()->nn_param.pad.back_size, + sizeof(uint32_t) * dim_num); + int32_t pad_value = op_->impl()->node()->nn_param.pad.const_val; + + if (!input_pv->IsAligned()) { + front_size = MapPadding(input_pv->AsStdVec(), front_size); + back_size = MapPadding(input_pv->AsStdVec(), back_size); + } + + auto pad = context_->infer_graph_->CreateOperation( + front_size, back_size, pad_value); + auto out_infer = CreateOutputsTensor(input_pv); + (*pad).BindInput(context_->GetMapedTensor(i_src)); + (*pad).BindOutput(out_infer[0]); + context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], input_pv); + next_tensors.push_back(op_->impl()->OutputsTensor()[0]); + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/transform/ops/reduce_layout_inference.h b/src/tim/transform/ops/reduce_layout_inference.h index c6cf4c4..e773c8f 100644 --- a/src/tim/transform/ops/reduce_layout_inference.h +++ b/src/tim/transform/ops/reduce_layout_inference.h @@ -36,6 +36,7 @@ namespace tim { namespace transform { template class ReduceLayoutInfer : public OpLayoutInfer { + public: ReduceLayoutInfer( const std::shared_ptr op, std::shared_ptr& context) @@ -60,7 +61,7 @@ class ReduceLayoutInfer : public OpLayoutInfer { (*reduce).BindInput(context_->GetMapedTensor(t_src)); if (op_->impl()->node()->nn_param.reduce.keep_dim) { auto otensor_infer = CreateOutputsTensor(pv); - (*reduce).BindOuput(otensor_infer[0]); + (*reduce).BindOutput(otensor_infer[0]); context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], pv); } else { auto out_pv = MakeShared(pv->Rank() - unique_axis.size()); @@ -87,6 +88,7 @@ using ReduceMaxLayoutInfer = ReduceLayoutInfer; using ReduceAnyLayoutInfer = ReduceLayoutInfer; using ReduceProdLayoutInfer = ReduceLayoutInfer; using ReduceMeanLayoutInfer = ReduceLayoutInfer; +using ReduceSumLayoutInfer = ReduceLayoutInfer; } // namespace transform } // namespace tim diff --git a/src/tim/transform/ops/space2batch_layout_inference.h b/src/tim/transform/ops/space2batch_layout_inference.h new file mode 100644 index 0000000..4d1886f --- /dev/null +++ b/src/tim/transform/ops/space2batch_layout_inference.h @@ -0,0 +1,84 @@ +/**************************************************************************** + * + * Copyright (c) 2020 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_SPACE2BATCH_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_SPACE2BATCH_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/space2batch.h" + +#include "src/tim/transform/ops/op_layout_inference.h" +#include "src/tim/transform/permute_vector.h" +#include "src/tim/vx/operation_private.h" +namespace tim { +namespace transform { +class SpaceToBatchLayoutInfer : public OpLayoutInfer { + public: + SpaceToBatchLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + void OnInputs( + std::vector>& next_tensors) override { + vx::DataLayout layout = op_->impl()->layout_; + auto required_pv = MakeShared(4); + if (layout == vx::DataLayout::CWHN) { + required_pv = std::make_shared>(kCWHN2WHCN); + } + auto input_tensors = op_->impl()->InputsTensor(); + + auto pv = context_->GetPermuteVector(input_tensors[0]); + auto final_pv = pv->Reverse()->Add(required_pv); + if (!final_pv->IsAligned()) { + auto perm_out = + InsertPermute(context_->GetMapedTensor(input_tensors[0]), final_pv); + context_->UpdateTensorMap(input_tensors[0], perm_out); + context_->SetPermuteVector(input_tensors[0], required_pv); + } + + uint32_t block_size_num = + op_->impl()->node()->nn_param.space2batch.block_size_num; + std::vector block_size(block_size_num); + memcpy(block_size.data(), + op_->impl()->node()->nn_param.space2batch.block_size, + sizeof(int) * block_size_num); + std::vector pad(4); + memcpy(pad.data(), op_->impl()->node()->nn_param.space2batch.pad, + sizeof(int) * 4); + + auto space2batch = + context_->infer_graph_->CreateOperation( + block_size, pad, vx::DataLayout::WHCN); + auto out_tensor_infer = CreateOutputsTensor(required_pv); + (*space2batch).BindInput(context_->GetMapedTensor(input_tensors[0])); + (*space2batch).BindOutput(out_tensor_infer[0]); + context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], required_pv); + // Add out tensor of src_graph into next_tensor + next_tensors.push_back(op_->impl()->OutputsTensor()[0]); + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/transform/ops/space2depth_layout_inference.h b/src/tim/transform/ops/space2depth_layout_inference.h new file mode 100644 index 0000000..222098d --- /dev/null +++ b/src/tim/transform/ops/space2depth_layout_inference.h @@ -0,0 +1,78 @@ +/**************************************************************************** + * + * Copyright (c) 2020 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_SPACE2DEPTH_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_SPACE2DEPTH_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/space2depth.h" + +#include "src/tim/transform/ops/op_layout_inference.h" +#include "src/tim/transform/permute_vector.h" +#include "src/tim/vx/operation_private.h" +namespace tim { +namespace transform { +class SpaceToDepthLayoutInfer : public OpLayoutInfer { + public: + SpaceToDepthLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + void OnInputs( + std::vector>& next_tensors) override { + vx::DataLayout layout = op_->impl()->layout_; + auto required_pv = MakeShared(4); + if (layout == vx::DataLayout::CWHN) { + required_pv = std::make_shared>(kCWHN2WHCN); + } + auto input_tensors = op_->impl()->InputsTensor(); + + auto pv = context_->GetPermuteVector(input_tensors[0]); + auto final_pv = pv->Reverse()->Add(required_pv); + if (!final_pv->IsAligned()) { + auto perm_out = + InsertPermute(context_->GetMapedTensor(input_tensors[0]), final_pv); + context_->UpdateTensorMap(input_tensors[0], perm_out); + context_->SetPermuteVector(input_tensors[0], required_pv); + } + + std::vector block_size = { + op_->impl()->node()->nn_param.space2depth.block_size[0], + op_->impl()->node()->nn_param.space2depth.block_size[1]}; + + auto space2depth = + context_->infer_graph_->CreateOperation( + block_size, vx::DataLayout::WHCN); + auto out_tensor_infer = CreateOutputsTensor(required_pv); + (*space2depth).BindInput(context_->GetMapedTensor(input_tensors[0])); + (*space2depth).BindOutput(out_tensor_infer[0]); + context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], required_pv); + // Add out tensor of src_graph into next_tensor + next_tensors.push_back(op_->impl()->OutputsTensor()[0]); + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/vx/ops/batch2space.cc b/src/tim/vx/ops/batch2space.cc index a567336..2eddc2e 100644 --- a/src/tim/vx/ops/batch2space.cc +++ b/src/tim/vx/ops/batch2space.cc @@ -30,9 +30,9 @@ namespace tim { namespace vx { namespace ops { -Batch2Space::Batch2Space(Graph* graph, const std::vector& block_size, - const std::vector& crop) - : Operation(graph, VSI_NN_OP_BATCH2SPACE), +BatchToSpace::BatchToSpace(Graph* graph, const std::vector& block_size, + const std::vector& crop, DataLayout layout) + : Operation(graph, VSI_NN_OP_BATCH2SPACE, 0, 0, layout), block_size_(block_size), crop_(crop) { this->impl()->node()->nn_param.batch2space.block_size = block_size_.data(); diff --git a/src/tim/vx/ops/depth2space.cc b/src/tim/vx/ops/depth2space.cc index 7a84d79..eda6bb6 100644 --- a/src/tim/vx/ops/depth2space.cc +++ b/src/tim/vx/ops/depth2space.cc @@ -30,8 +30,9 @@ namespace tim { namespace vx { namespace ops { -DepthToSpace::DepthToSpace(Graph* graph, int block_size) - : Operation(graph, VSI_NN_OP_DEPTH2SPACE), block_size_(block_size) { +DepthToSpace::DepthToSpace(Graph* graph, int block_size, DataLayout layout) + : Operation(graph, VSI_NN_OP_DEPTH2SPACE, 0, 0, layout), + block_size_(block_size) { this->impl()->node()->nn_param.depth2space.block_size = block_size_; } } // namespace ops diff --git a/src/tim/vx/ops/reduce.cc b/src/tim/vx/ops/reduce.cc index 9af8421..f424e3e 100644 --- a/src/tim/vx/ops/reduce.cc +++ b/src/tim/vx/ops/reduce.cc @@ -47,6 +47,7 @@ DEFINE_REDUCE_OP(Max, VSI_NN_REDUCE_MAX); DEFINE_REDUCE_OP(Any, VSI_NN_REDUCE_ANY); DEFINE_REDUCE_OP(Prod, VSI_NN_REDUCE_PROD); DEFINE_REDUCE_OP(Mean, VSI_NN_REDUCE_MEAN); +DEFINE_REDUCE_OP(Sum, VSI_NN_REDUCE_SUM); #undef DEFINE_REDUCE_OP diff --git a/src/tim/vx/ops/space2batch.cc b/src/tim/vx/ops/space2batch.cc index 2c990b9..a171a2f 100644 --- a/src/tim/vx/ops/space2batch.cc +++ b/src/tim/vx/ops/space2batch.cc @@ -30,9 +30,9 @@ namespace tim { namespace vx { namespace ops { -Space2Batch::Space2Batch(Graph* graph, const std::vector& block_size, - const std::vector& pad) - : Operation(graph, VSI_NN_OP_SPACE2BATCH), +SpaceToBatch::SpaceToBatch(Graph* graph, const std::vector& block_size, + const std::vector& pad, DataLayout layout) + : Operation(graph, VSI_NN_OP_SPACE2BATCH, 0, 0, layout), block_size_(block_size), pad_(pad) { this->impl()->node()->nn_param.space2batch.block_size = block_size_.data(); diff --git a/src/tim/vx/ops/space2depth.cc b/src/tim/vx/ops/space2depth.cc index 4ad0303..11e300a 100644 --- a/src/tim/vx/ops/space2depth.cc +++ b/src/tim/vx/ops/space2depth.cc @@ -30,8 +30,10 @@ namespace tim { namespace vx { namespace ops { -SpaceToDepth::SpaceToDepth(Graph* graph, std::vector block_size) - : Operation(graph, VSI_NN_OP_SPACE2DEPTH), block_size_(block_size) { +SpaceToDepth::SpaceToDepth(Graph* graph, std::vector block_size, + DataLayout layout) + : Operation(graph, VSI_NN_OP_SPACE2DEPTH, 0, 0, layout), + block_size_(block_size) { this->impl()->node()->nn_param.space2depth.block_size[0] = block_size_[0]; this->impl()->node()->nn_param.space2depth.block_size[1] = block_size_[1]; }