diff --git a/include/tim/vx/ops/batchnorm.h b/include/tim/vx/ops/batchnorm.h index 30d25d4..6feb08d 100644 --- a/include/tim/vx/ops/batchnorm.h +++ b/include/tim/vx/ops/batchnorm.h @@ -42,7 +42,7 @@ namespace ops { class BatchNorm : public Operation { public: - BatchNorm(Graph* graph, float eps); + BatchNorm(Graph* graph, float eps, DataLayout input_layout = DataLayout::WHCN); std::shared_ptr Clone(std::shared_ptr& graph) const override; diff --git a/src/tim/transform/layout_inference.cc b/src/tim/transform/layout_inference.cc index a9d0547..088c012 100644 --- a/src/tim/transform/layout_inference.cc +++ b/src/tim/transform/layout_inference.cc @@ -57,6 +57,7 @@ #include "ops/logical_layout_inference.h" #include "ops/arg_layout_inference.h" #include "ops/deconv2d_layout_inference.h" +#include "ops/batchnorm_layout_inference.h" #include "ops/default_layout_inference.h" #include @@ -255,6 +256,7 @@ std::vector> HandleLayoutInfer( REGIST_LAYOUT_INFERENCE(VSI_NN_OP_ARGMAX, Arg); REGIST_LAYOUT_INFERENCE(VSI_NN_OP_ARGMIN, Arg); REGIST_LAYOUT_INFERENCE(VSI_NN_OP_DECONVOLUTION, DeConv2d); + REGIST_LAYOUT_INFERENCE(VSI_NN_OP_BATCH_NORM, BatchNorm); REGIST_LOGICAL_LAYOUT_INFERENCE(VSI_NN_OP_LOGICAL_OPS); REGIST_REDUCE_LAYOUT_INFERENCE(VSI_NN_OP_REDUCE); // use default layout inference diff --git a/src/tim/transform/ops/batchnorm_layout_inference.h b/src/tim/transform/ops/batchnorm_layout_inference.h new file mode 100644 index 0000000..b1e7518 --- /dev/null +++ b/src/tim/transform/ops/batchnorm_layout_inference.h @@ -0,0 +1,90 @@ +/**************************************************************************** + * + * Copyright (c) 2020 Vivante Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + *****************************************************************************/ +#ifndef TIM_LAYOUT_INFER_BATCHNORM_LAYOUT_INFERENCE_H_ +#define TIM_LAYOUT_INFER_BATCHNORM_LAYOUT_INFERENCE_H_ + +#include "tim/vx/ops/batchnorm.h" + +#include "ops/op_layout_inference.h" +#include "permute_vector.h" +#include "operation_private.h" +namespace tim { +namespace transform { +class BatchNormLayoutInfer : public OpLayoutInfer { + public: + BatchNormLayoutInfer( + const std::shared_ptr op, + std::shared_ptr& context) + : OpLayoutInfer(op, context) {} + + void OnInputs( + std::vector>& next_tensors) override { + vx::DataLayout layout = op_->impl()->layout_; + auto required_pv = MakeShared(4); + if (layout == vx::DataLayout::CWHN) { + required_pv = std::make_shared>(kCWHN2WHCN); + } + auto input_tensors = op_->impl()->InputsTensor(); + assert(input_tensors.size() == 5); + for (uint32_t idx = 0; idx < input_tensors.size(); idx++) { + std::shared_ptr perm_out; + std::shared_ptr input_pv; + auto src_in = input_tensors[idx]; + if (src_in->IsConstTensor()) { + perm_out = context_->infer_graph_->CreateTensor(src_in->GetSpec(), src_in->GetDataRef()); + input_pv = MakeShared(src_in->GetShape().size()); + } else { + perm_out = context_->GetMapedTensor(src_in); + input_pv = context_->GetPermuteVector(src_in); + context_->SetPermuteVector(src_in, input_pv); + if (idx == 0) { + auto final_pv = input_pv->Reverse()->Add(required_pv); + if (!final_pv->IsAligned()) { + perm_out = InsertPermute(perm_out, required_pv); + context_->SetPermuteVector(src_in, required_pv); + } + } + } + context_->UpdateTensorMap(src_in, perm_out); + } + + auto batchnorm = op_->Clone(context_->infer_graph_); + auto out_tensor_infer = CreateOutputsTensor(required_pv); + (*batchnorm).BindInput(context_->GetMapedTensor(input_tensors[0])); + (*batchnorm).BindInput(context_->GetMapedTensor(input_tensors[1])); + (*batchnorm).BindInput(context_->GetMapedTensor(input_tensors[2])); + (*batchnorm).BindInput(context_->GetMapedTensor(input_tensors[3])); + (*batchnorm).BindInput(context_->GetMapedTensor(input_tensors[4])); + + (*batchnorm).BindOutput(out_tensor_infer[0]); + context_->SetPermuteVector(op_->impl()->OutputsTensor()[0], required_pv); + // Add out tensor of src_graph into next_tensor + next_tensors.push_back(op_->impl()->OutputsTensor()[0]); + } +}; + +} // namespace transform +} // namespace tim + +#endif \ No newline at end of file diff --git a/src/tim/vx/ops/batchnorm.cc b/src/tim/vx/ops/batchnorm.cc index fc221cc..3685259 100644 --- a/src/tim/vx/ops/batchnorm.cc +++ b/src/tim/vx/ops/batchnorm.cc @@ -30,8 +30,8 @@ namespace tim { namespace vx { namespace ops { -BatchNorm::BatchNorm(Graph* graph, float eps) - : Operation(graph, VSI_NN_OP_BATCH_NORM), eps_(eps) { +BatchNorm::BatchNorm(Graph* graph, float eps, DataLayout input_layout) + : Operation(graph, VSI_NN_OP_BATCH_NORM, 0, 0, input_layout), eps_(eps) { this->impl()->node()->nn_param.batch_norm.eps = eps_; } diff --git a/src/tim/vx/ops/batchnorm_test.cc b/src/tim/vx/ops/batchnorm_test.cc new file mode 100644 index 0000000..fe59d6a --- /dev/null +++ b/src/tim/vx/ops/batchnorm_test.cc @@ -0,0 +1,168 @@ +/**************************************************************************** +* +* Copyright (c) 2021 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ +#include "tim/vx/context.h" +#include "tim/vx/graph.h" +#include "tim/vx/ops/batchnorm.h" +#include "tim/transform/layout_inference.h" + +#include "gtest/gtest.h" +#include "test_utils.h" + +TEST(BatchNorm, shape_3_3_2_1_fp32_cwhn) { + auto ctx = tim::vx::Context::Create(); + auto graph = ctx->CreateGraph(); + + tim::vx::ShapeType in_shape({2, 3, 3, 1}); + tim::vx::ShapeType out_shape({2, 3, 3, 1}); + tim::vx::ShapeType mean_shape({2}); + tim::vx::ShapeType var_shape({2}); + tim::vx::ShapeType gamma_shape({2}); + tim::vx::ShapeType beta_shape({2}); + tim::vx::TensorSpec input_spec(tim::vx::DataType::FLOAT32, + in_shape, tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec mean_spec(tim::vx::DataType::FLOAT32, mean_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec var_spec(tim::vx::DataType::FLOAT32, var_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec gamma_spec(tim::vx::DataType::FLOAT32, gamma_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec beta_spec(tim::vx::DataType::FLOAT32, beta_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec output_spec(tim::vx::DataType::FLOAT32, + out_shape, tim::vx::TensorAttribute::OUTPUT); + + std::vector in_data = { + 0.59885779, 0.62662862, 0.63011179, 0.82569427, 0.64772359, 0.42895413, + 0.30216458, 0.01351635, 0.32545444, 0.0360674, 0.33967769, 0.18092504, + 0.09479915, 0.52258112, 0.46735646, 0.95689111, 0.51619059, 0.82685718}; + std::vector golden = { + 0.92227477, 0.40612271, 1.09906762, + 1.00176775, 1.19869136, -0.18535967, + -0.7560139, -1.42843423, -0.62427138, + -1.3609569, -0.54381545, -0.92751329, + -1.92900686, 0.09479138, 0.17841823, + 1.39433545, 0.45465564, 1.0052474, + }; + + std::vector mean_data = { + 0.43581513, 0.49090168 + }; + std::vector var_data = { + 0.03025229, 0.11069085 + }; + std::vector gamma_data = { + 1,1 + }; + std::vector beta_data = { + 0, 0 + }; + + auto input_tensor = graph->CreateTensor(input_spec); + auto output_tensor = graph->CreateTensor(output_spec); + auto mean = graph->CreateTensor(mean_spec, mean_data.data()); + auto var = graph->CreateTensor(var_spec, var_data.data()); + auto gamma = graph->CreateTensor(gamma_spec, gamma_data.data()); + auto beta = graph->CreateTensor(beta_spec, beta_data.data()); + + float epsilon = 0.001; + + auto op = graph->CreateOperation(epsilon, tim::vx::DataLayout::CWHN); + (*op).BindInputs({input_tensor, mean, var,gamma, beta}).BindOutputs({output_tensor}); + + auto final_graph = tim::transform::LayoutInference(graph, ctx); + + EXPECT_TRUE(final_graph.first->Compile()); + final_graph.second[input_tensor]->CopyDataToTensor( + in_data.data(), in_data.size() * sizeof(float)); + EXPECT_TRUE(final_graph.first->Run()); + + std::vector output(golden.size()); + EXPECT_TRUE(final_graph.second[output_tensor]->CopyDataFromTensor(output.data())); + + for (uint32_t idx = 0; idx < golden.size(); idx++) { + EXPECT_TRUE(std::abs(golden[idx] - output[idx]) < 0.01); + } +} + +TEST(BatchNorm, shape_3_3_2_1_fp32_whcn) { + auto ctx = tim::vx::Context::Create(); + auto graph = ctx->CreateGraph(); + + tim::vx::ShapeType in_shape({3, 3, 2, 1}); + tim::vx::ShapeType out_shape({3, 3, 2, 1}); + tim::vx::ShapeType mean_shape({2}); + tim::vx::ShapeType var_shape({2}); + tim::vx::ShapeType gamma_shape({2}); + tim::vx::ShapeType beta_shape({2}); + tim::vx::TensorSpec input_spec(tim::vx::DataType::FLOAT32, + in_shape, tim::vx::TensorAttribute::INPUT); + tim::vx::TensorSpec mean_spec(tim::vx::DataType::FLOAT32, mean_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec var_spec(tim::vx::DataType::FLOAT32, var_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec gamma_spec(tim::vx::DataType::FLOAT32, gamma_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec beta_spec(tim::vx::DataType::FLOAT32, beta_shape, tim::vx::CONSTANT); + tim::vx::TensorSpec output_spec(tim::vx::DataType::FLOAT32, + out_shape, tim::vx::TensorAttribute::OUTPUT); + + std::vector in_data = { + 0.598858, 0.630112, 0.647724, 0.302165, 0.325454, 0.339678, + 0.094799, 0.467356, 0.516191, 0.626629, 0.825694, 0.428954, + 0.013516, 0.036067, 0.180925, 0.522581, 0.956891, 0.826857}; + std::vector golden = { + 0.922275, 1.099068, 1.198692, -0.756014, -0.624271, -0.543815, + -1.929007, 0.178418, 0.454656, 0.406123, 1.001768, -0.185360, + -1.428434, -1.360957, -0.927513, 0.094791, 1.394335, 1.005247}; + + std::vector mean_data = { + 0.43581513, 0.49090168 + }; + std::vector var_data = { + 0.03025229, 0.11069085 + }; + std::vector gamma_data = { + 1,1 + }; + std::vector beta_data = { + 0, 0 + }; + + auto input_tensor = graph->CreateTensor(input_spec); + auto output_tensor = graph->CreateTensor(output_spec); + auto mean = graph->CreateTensor(mean_spec, mean_data.data()); + auto var = graph->CreateTensor(var_spec, var_data.data()); + auto gamma = graph->CreateTensor(gamma_spec, gamma_data.data()); + auto beta = graph->CreateTensor(beta_spec, beta_data.data()); + + float epsilon = 0.001; + + auto op = graph->CreateOperation(epsilon); + (*op).BindInputs({input_tensor, mean, var,gamma, beta}).BindOutputs({output_tensor}); + + EXPECT_TRUE(graph->Compile()); + input_tensor->CopyDataToTensor(in_data.data(), + in_data.size() * sizeof(float)); + EXPECT_TRUE(graph->Run()); + + std::vector output(golden.size()); + EXPECT_TRUE(output_tensor->CopyDataFromTensor(output.data())); + + for (uint32_t idx = 0; idx < golden.size(); idx++) { + EXPECT_TRUE(std::abs(golden[idx] - output[idx]) < 0.01); + } +} \ No newline at end of file