[NNAdapter][TIM-VX]add batch_norm, scale, conv2d_transpose, nearest_interp, bilinear_interp, matmul, flatten_contiguous_range

lite/backends/nnadapter/nnadapter/driver/verisilicon_timvx/converter/all.h

@@ -17,16 +17,21 @@
 
 REGISTER_CONVERTER(ADD, ConvertElementwise)
 REGISTER_CONVERTER(AVERAGE_POOL_2D, ConvertPool2D)
+REGISTER_CONVERTER(BATCH_NORMALIZATION, ConvertBatchNormalization)
 REGISTER_CONVERTER(CONCAT, ConvertConcat)
 REGISTER_CONVERTER(CONV_2D, ConvertConv2D)
+REGISTER_CONVERTER(CONV_2D_TRANSPOSE, ConvertConv2DTranspose)
 REGISTER_CONVERTER(DIV, ConvertElementwise)
 REGISTER_CONVERTER(FLATTEN, ConvertFlatten)
 REGISTER_CONVERTER(FULLY_CONNECTED, ConvertFullyConnected)
 REGISTER_CONVERTER(MAX_POOL_2D, ConvertPool2D)
+REGISTER_CONVERTER(MAT_MUL, ConvertMatMul)
 REGISTER_CONVERTER(MUL, ConvertElementwise)
 REGISTER_CONVERTER(RELU, ConvertUnaryActivations)
 REGISTER_CONVERTER(RELU6, ConvertUnaryActivations)
 REGISTER_CONVERTER(RESHAPE, ConvertReshape)
+REGISTER_CONVERTER(RESIZE_NEAREST, ConvertResizeNearest)
+REGISTER_CONVERTER(RESIZE_LINEAR, ConvertResizeLinear)
 REGISTER_CONVERTER(SIGMOID, ConvertUnaryActivations)
 REGISTER_CONVERTER(SOFTMAX, ConvertSoftmax)
 REGISTER_CONVERTER(SUB, ConvertElementwise)

lite/backends/nnadapter/nnadapter/driver/verisilicon_timvx/converter/batch_normalization.cc

@@ -0,0 +1,50 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "core/operation/batch_normalization.h"
+#include "driver/verisilicon_timvx/converter/converter.h"
+#include "utility/debug.h"
+#include "utility/logging.h"
+
+namespace nnadapter {
+namespace verisilicon_timvx {
+
+int ConvertBatchNormalization(Converter* converter, hal::Operation* operation) {
+  BATCH_NORMALIZATION_OPERATION_EXTRACT_INPUTS_OUTPUTS
+
+  // Convert to tim-vx tensors and operators
+  auto input_operator = converter->GetMappedTensor(input_operand);
+  if (!input_operator) {
+    input_operator = converter->ConvertOperand(input_operand);
+  }
+
+  auto scale_operator = converter->ConvertOperand(scale_operand);
+  auto offset_operator = converter->ConvertOperand(bias_operand);
+  auto mean_operator = converter->ConvertOperand(mean_operand);
+  auto variance_operator = converter->ConvertOperand(variance_operand);
+  auto output_tensor = converter->ConvertOperand(output_operand);
+
+  auto batch_norm_op =
+      converter->graph()->CreateOperation<tim::vx::ops::BatchNorm>(epsilon);
+  batch_norm_op->BindInputs({input_operator,
+                             mean_operator,
+                             variance_operator,
+                             scale_operator,
+                             offset_operator});
+  batch_norm_op->BindOutputs({output_tensor});
+  return NNADAPTER_NO_ERROR;
+}
+
+}  // namespace verisilicon_timvx
+}  // namespace nnadapter

lite/backends/nnadapter/nnadapter/driver/verisilicon_timvx/converter/conv2d_transpose.cc

@@ -0,0 +1,95 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "core/operation/conv2d_transpose.h"
+#include "core/operation/conv2d.h"
+#include "driver/verisilicon_timvx/converter/converter.h"
+#include "utility/debug.h"
+#include "utility/logging.h"
+
+namespace nnadapter {
+namespace verisilicon_timvx {
+
+int ConvertConv2DTranspose(Converter* converter, hal::Operation* operation) {
+  CONV_2D_TRANSPOSE_OPERATION_EXTRACT_INPUTS_OUTPUTS
+  // Dynamic shapes are still not supported
+  NNADAPTER_CHECK_EQ(input_operand->type.dimensions.dynamic_count, 0);
+  if (auto_pad != NNADAPTER_AUTO_PAD_NONE) {
+    operation::UpdateConv2DPadAndDilation(
+        input_operand->type.dimensions.data[2],
+        filter_height,
+        auto_pad,
+        &pad_height_top,
+        &pad_height_bottom,
+        stride_height,
+        &dilation_height);
+    operation::UpdateConv2DPadAndDilation(
+        input_operand->type.dimensions.data[3],
+        filter_width,
+        auto_pad,
+        &pad_width_left,
+        &pad_width_right,
+        stride_width,
+        &dilation_width);
+  }
+  // Convert to tim-vx tensors and operators
+  auto input_tensor = converter->GetMappedTensor(input_operand);
+  if (!input_tensor) {
+    input_tensor = converter->ConvertOperand(input_operand);
+  }
+  int32_t multiplier = 0;
+  std::vector<int32_t> filter_dimensions(
+      filter_operand->type.dimensions.data,
+      filter_operand->type.dimensions.data +
+          filter_operand->type.dimensions.count);
+  if (is_depthwise_mode) {
+    multiplier = output_channel_size / group;
+    NNADAPTER_CHECK_GT(filter_operand->type.dimensions.count, 2);
+    // Oc,1,H,W -> 1,Oc,H,W
+    filter_dimensions[0] = filter_dimensions[1];
+    filter_dimensions[1] = output_channel_size;
+  }
+  auto filter_tensor =
+      converter->ConvertOperand(filter_operand, filter_dimensions);
+  auto bias_tensor = converter->ConvertOperand(bias_operand);
+  if (!bias_tensor) {
+    bias_tensor = converter->ConvertOperand(bias_operand);
+  }
+  auto output_tensor = converter->ConvertOperand(output_operand);
+  auto conv2dtranspose_op =
+      converter->graph()->CreateOperation<tim::vx::ops::DeConv2d>(
+          filter_dimensions[0],
+          tim::vx::PadType::AUTO,
+          std::array<uint32_t, 2>({static_cast<uint32_t>(filter_width),
+                                   static_cast<uint32_t>(filter_height)}),
+          std::array<uint32_t, 2>({static_cast<uint32_t>(stride_width),
+                                   static_cast<uint32_t>(stride_height)}),
+          std::array<uint32_t, 2>(
+              {static_cast<uint32_t>(output_padding_width),
+               static_cast<uint32_t>(output_padding_height)}),
+          std::array<uint32_t, 4>({static_cast<uint32_t>(pad_width_left),
+                                   static_cast<uint32_t>(pad_width_right),
+                                   static_cast<uint32_t>(pad_height_top),
+                                   static_cast<uint32_t>(pad_height_bottom)}),
+          group);
+  conv2dtranspose_op->BindInputs({input_tensor, filter_tensor, bias_tensor});
+  conv2dtranspose_op->BindOutputs({output_tensor});
+  NNADAPTER_CHECK_EQ(fuse_code, NNADAPTER_FUSED_NONE)
+      << "Missing the processing of fuse_code(" << fuse_code
+      << ") in unpack_op_fusion.cc";
+  return NNADAPTER_NO_ERROR;
+}
+
+}  // namespace verisilicon_timvx
+}  // namespace nnadapter

lite/backends/nnadapter/nnadapter/driver/verisilicon_timvx/converter/mat_mul.cc

@@ -0,0 +1,48 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "core/operation/mat_mul.h"
+#include "driver/verisilicon_timvx/converter/converter.h"
+#include "utility/debug.h"
+#include "utility/logging.h"
+
+namespace nnadapter {
+namespace verisilicon_timvx {
+
+int ConvertMatMul(Converter* converter, hal::Operation* operation) {
+  MAT_MUL_OPERATION_EXTRACT_INPUTS_OUTPUTS
+  NNADAPTER_CHECK_NE(x_operand->type.dimensions.count, 1);
+  NNADAPTER_CHECK_NE(y_operand->type.dimensions.count, 1);
+
+  // Convert to tim-vx tensors and operators
+  auto x_tensor = converter->GetMappedTensor(x_operand);
+  if (!x_tensor) {
+    x_tensor = converter->ConvertOperand(x_operand);
+  }
+
+  auto y_tensor = converter->GetMappedTensor(y_operand);
+  if (!y_tensor) {
+    y_tensor = converter->ConvertOperand(y_operand);
+  }
+  auto output_tensor = converter->ConvertOperand(output_operand);
+
+  auto mat_mul_op = converter->graph()->CreateOperation<tim::vx::ops::Matmul>(
+      transpose_x, transpose_y, false, false);
+  mat_mul_op->BindInputs({x_tensor, y_tensor});
+  mat_mul_op->BindOutputs({output_tensor});
+  return NNADAPTER_NO_ERROR;
+}
+
+}  // namespace verisilicon_timvx
+}  // namespace nnadapter

lite/backends/nnadapter/nnadapter/driver/verisilicon_timvx/converter/resize_linear.cc

@@ -0,0 +1,47 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "core/operation/resize_linear.h"
+#include "driver/verisilicon_timvx/converter/converter.h"
+#include "utility/debug.h"
+#include "utility/logging.h"
+
+namespace nnadapter {
+namespace verisilicon_timvx {
+
+int ConvertResizeLinear(Converter* converter, hal::Operation* operation) {
+  RESIZE_LINEAR_OPERATION_EXTRACT_INPUTS_OUTPUTS
+
+  // Convert to tim-vx tensors and operators
+  auto input_tensor = converter->GetMappedTensor(input_operand);
+  if (!input_tensor) {
+    input_tensor = converter->ConvertOperand(input_operand);
+  }
+  auto output_tensor = converter->ConvertOperand(output_operand);
+
+  float factor = reinterpret_cast<float*>(input_operands[2]->buffer)[0];
+  auto resize_op = converter->graph()->CreateOperation<tim::vx::ops::Resize>(
+      tim::vx::ResizeType::BILINEAR,
+      factor,
+      align_corners,
+      false,
+      output_operand->type.dimensions.data[2],
+      output_operand->type.dimensions.data[3]);
+  resize_op->BindInputs({input_tensor});
+  resize_op->BindOutputs({output_tensor});
+  return NNADAPTER_NO_ERROR;
+}
+
+}  // namespace verisilicon_timvx
+}  // namespace nnadapter

lite/backends/nnadapter/nnadapter/driver/verisilicon_timvx/converter/resize_nearest.cc

@@ -0,0 +1,47 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "core/operation/resize_nearest.h"
+#include "driver/verisilicon_timvx/converter/converter.h"
+#include "utility/debug.h"
+#include "utility/logging.h"
+
+namespace nnadapter {
+namespace verisilicon_timvx {
+
+int ConvertResizeNearest(Converter* converter, hal::Operation* operation) {
+  RESIZE_NEAREST_OPERATION_EXTRACT_INPUTS_OUTPUTS
+
+  // Convert to tim-vx tensors and operators
+  auto input_tensor = converter->GetMappedTensor(input_operand);
+  if (!input_tensor) {
+    input_tensor = converter->ConvertOperand(input_operand);
+  }
+  auto output_tensor = converter->ConvertOperand(output_operand);
+
+  float factor = reinterpret_cast<float*>(input_operands[2]->buffer)[0];
+  auto resize_op = converter->graph()->CreateOperation<tim::vx::ops::Resize>(
+      tim::vx::ResizeType::NEAREST_NEIGHBOR,
+      factor,
+      align_corners,
+      false,
+      output_operand->type.dimensions.data[2],
+      output_operand->type.dimensions.data[3]);
+  resize_op->BindInputs({input_tensor});
+  resize_op->BindOutputs({output_tensor});
+  return NNADAPTER_NO_ERROR;
+}
+
+}  // namespace verisilicon_timvx
+}  // namespace nnadapter

lite/backends/nnadapter/nnadapter/optimizer/symm2asymm.cc

@@ -99,10 +99,13 @@ NNADAPTER_EXPORT void ConvertQuantizationSymmToAsymm(hal::Model* model) {
         ConvertOperandSymmToAsymm(output_operands[0], 128);
       } break;
       case NNADAPTER_AVERAGE_POOL_2D:
+      case NNADAPTER_BATCH_NORMALIZATION:
       case NNADAPTER_MAX_POOL_2D:
       case NNADAPTER_RELU:
       case NNADAPTER_RELU6:
       case NNADAPTER_RESHAPE:
+      case NNADAPTER_RESIZE_NEAREST:
+      case NNADAPTER_RESIZE_LINEAR:
       case NNADAPTER_TANH:
       case NNADAPTER_FLATTEN:
       case NNADAPTER_TRANSPOSE:

lite/kernels/nnadapter/converter/all.h

@@ -15,7 +15,10 @@
 #ifndef __NNADAPTER_CONVERTER_ALL_H__  // NOLINT
 #define __NNADAPTER_CONVERTER_ALL_H__
 
-REGISTER_CONVERTER(batch_norm, ConvertBatchNorm, "huawei_ascend_npu");
+REGISTER_CONVERTER(batch_norm,
+                   ConvertBatchNorm,
+                   "huawei_ascend_npu,verisilicon_"
+                   "timvx");
 REGISTER_CONVERTER(cast, ConvertCast, "huawei_ascend_npu");
 REGISTER_CONVERTER(clip, ConvertClip, "huawei_ascend_npu");
 REGISTER_CONVERTER(conv2d,
@@ -34,9 +37,11 @@ REGISTER_CONVERTER(pool2d,
                    "rockchip_npu,mediatek_apu,huawei_kirin_npu,huawei_ascend_"
                    "npu,amlogic_npu,imagination_nna,cambricon_mlu,verisilicon_"
                    "timvx");
-REGISTER_CONVERTER(matmul,
-                   ConvertMatmul,
-                   "huawei_ascend_npu,huawei_kirin_npu,imagination_nna");
+REGISTER_CONVERTER(
+    matmul,
+    ConvertMatmul,
+    "huawei_ascend_npu,huawei_kirin_npu,imagination_nna,verisilicon"
+    "_timvx");
 REGISTER_CONVERTER(matmul_v2,
                    ConvertMatmulV2,
                    "huawei_ascend_npu,huawei_kirin_npu,imagination_nna");
@@ -48,7 +53,7 @@ REGISTER_CONVERTER(softmax,
 REGISTER_CONVERTER(cumsum, ConvertCumsum, "huawei_ascend_npu");
 REGISTER_CONVERTER(conv2d_transpose,
                    ConvertConv2dTranspose,
-                   "huawei_ascend_npu,amlogic_npu");
+                   "huawei_ascend_npu,amlogic_npu,verisilicon_timvx");
 REGISTER_CONVERTER(reshape,
                    ConvertReshape,
                    "rockchip_npu,mediatek_apu,huawei_kirin_npu,huawei_ascend_"
@@ -201,10 +206,22 @@ REGISTER_CONVERTER(concat,
                    "npu,amlogic_npu,verisilicon_timvx");
 REGISTER_CONVERTER(split, ConvertSplit, "huawei_kirin_npu,huawei_ascend_npu");
 REGISTER_CONVERTER(calib, ConvertCalib, "huawei_ascend_npu");
-REGISTER_CONVERTER(nearest_interp, ConvertInterpolate, "huawei_ascend_npu");
-REGISTER_CONVERTER(nearest_interp_v2, ConvertInterpolate, "huawei_ascend_npu");
-REGISTER_CONVERTER(bilinear_interp, ConvertInterpolate, "huawei_ascend_npu");
-REGISTER_CONVERTER(bilinear_interp_v2, ConvertInterpolate, "huawei_ascend_npu");
+REGISTER_CONVERTER(nearest_interp,
+                   ConvertInterpolate,
+                   "huawei_ascend_npu,"
+                   "verisilicon_timvx");
+REGISTER_CONVERTER(nearest_interp_v2,
+                   ConvertInterpolate,
+                   "huawei_ascend_npu,"
+                   "verisilicon_timvx");
+REGISTER_CONVERTER(bilinear_interp,
+                   ConvertInterpolate,
+                   "huawei_ascend_npu,"
+                   "verisilicon_timvx");
+REGISTER_CONVERTER(bilinear_interp_v2,
+                   ConvertInterpolate,
+                   "huawei_ascend_npu,"
+                   "verisilicon_timvx");
 REGISTER_CONVERTER(flatten,
                    ConvertFlatten,
                    "rockchip_npu,mediatek_apu,huawei_kirin_npu,huawei_ascend_"

lite/tests/kernels/batch_norm_compute_test.cc

@@ -166,6 +166,8 @@ TEST(BatchNorm, precision) {
   abs_error = 1e-2;
   // TODO(shentanyue): support later
   return;
+#elif defined(NNADAPTER_WITH_VERISILICON_TIMVX)
+  abs_error = 1e-2;
 #else
   return;
 #endif