From 46dec52c365525ba1a2a45672d2e782c75ccd326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Sat, 4 Jun 2022 07:05:49 -0700 Subject: [PATCH 01/27] add method for post ops --- paddle/fluid/operators/matmul_op.cc | 3 +- .../operators/mkldnn/matmul_mkldnn_op.cc | 39 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/paddle/fluid/operators/matmul_op.cc b/paddle/fluid/operators/matmul_op.cc index 9d381e1f22b5f9..cd72d2450c1117 100644 --- a/paddle/fluid/operators/matmul_op.cc +++ b/paddle/fluid/operators/matmul_op.cc @@ -739,7 +739,8 @@ class MatMulOpMaker : public framework::OpProtoAndCheckerMaker { R"DOC(If true, use the transpose of `Y`. )DOC") .SetDefault(false); - AddAttr("alpha", "The scale of Out").SetDefault(1.0f); + AddAttr("alpha", "The scale of Out") + .SetDefault(1.0f); AddAttr( "use_mkldnn", "(bool, default false) Indicates if MKL-DNN kernel will be used") diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index e9abe84e679803..4ac7b663418cea 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -145,16 +145,10 @@ class MatMulMKLDNNHandler this->AcquireForwardPrimitiveDescriptor(attrs, x_md, y_md, out_md); } // Constructor for FWD MatMul - MatMulMKLDNNHandler(const dnnl::engine engine, const ExecutionContext& ctx, - float scale) + MatMulMKLDNNHandler(const dnnl::engine engine, const ExecutionContext& ctx) : paddle::platform::MKLDNNHandlerNoCachingT( engine, ctx.GetPlace()) { - dnnl::primitive_attr attr; - float scale_out = ComputeOutputScale(ctx); - if (scale_out != 1.0f) { - constexpr unsigned tensor_wide_scale = 0; - attr.set_output_scales(tensor_wide_scale, {scale_out}); - } + const dnnl::primitive_attr matmul_attrs = CreatePostOps(ctx); auto matmul_dims_ = GetMatmulDims(ctx); auto x_md = memory::desc(matmul_dims_.x_dims, MKLDNNGetDataType(), @@ -163,7 +157,7 @@ class MatMulMKLDNNHandler matmul_dims_.y_strides); auto out_md = memory::desc(matmul_dims_.out_dims, MKLDNNGetDataType(), matmul_dims_.out_strides); - this->AcquireForwardPrimitiveDescriptor(attr, x_md, y_md, out_md); + this->AcquireForwardPrimitiveDescriptor(matmul_attrs, x_md, y_md, out_md); } std::shared_ptr AcquireWeightsMemory(const Tensor* input) { @@ -427,6 +421,19 @@ class MatMulMKLDNNHandler return std::make_tuple(x_offset_, y_offset_, out_offset_); } + dnnl::primitive_attr CreatePostOps(const ExecutionContext& ctx) { + dnnl::primitive_attr matmul_attrs; + dnnl::post_ops post_operations; + + float scale_out = ComputeOutputScale(ctx); + if (scale_out != 1.0f) { + matmul_attrs.set_output_scales(0, {scale_out}); + } + + matmul_attrs.set_post_ops(post_operations); + return matmul_attrs; + } + private: uint32_t x_offset_; uint32_t y_offset_; @@ -497,23 +504,19 @@ static void ExecuteMatMul(const ExecutionContext& ctx) { auto* x = ctx.Input("X"); auto* y = ctx.Input("Y"); auto* out = ctx.Output("Out"); - float alpha = ctx.HasAttr("alpha") ? ctx.Attr("alpha") : 1.0f; const auto& dev_ctx = ctx.template device_context(); + const auto& onednn_engine = dev_ctx.GetEngine(); if (force_fp32_output || ((!is_int8) && (!is_bfloat16))) { - MatMulMKLDNNHandler(dev_ctx.GetEngine(), ctx, alpha) - .Execute(x, y, out); + MatMulMKLDNNHandler(onednn_engine, ctx).Execute(x, y, out); } else if (is_bfloat16) { - MatMulMKLDNNHandler(dev_ctx.GetEngine(), - ctx, alpha) + MatMulMKLDNNHandler(onednn_engine, ctx) .Execute(x, y, out); } else if (fuse_relu) { - MatMulMKLDNNHandler(dev_ctx.GetEngine(), ctx, alpha) - .Execute(x, y, out); + MatMulMKLDNNHandler(onednn_engine, ctx).Execute(x, y, out); } else { - MatMulMKLDNNHandler(dev_ctx.GetEngine(), ctx, alpha) - .Execute(x, y, out); + MatMulMKLDNNHandler(onednn_engine, ctx).Execute(x, y, out); } } From 14cf236a34e6dcc7943b3515d046bf54d5494c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Sat, 4 Jun 2022 07:09:41 -0700 Subject: [PATCH 02/27] format code --- paddle/fluid/operators/matmul_op.cc | 3 +-- paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/matmul_op.cc b/paddle/fluid/operators/matmul_op.cc index cd72d2450c1117..9d381e1f22b5f9 100644 --- a/paddle/fluid/operators/matmul_op.cc +++ b/paddle/fluid/operators/matmul_op.cc @@ -739,8 +739,7 @@ class MatMulOpMaker : public framework::OpProtoAndCheckerMaker { R"DOC(If true, use the transpose of `Y`. )DOC") .SetDefault(false); - AddAttr("alpha", "The scale of Out") - .SetDefault(1.0f); + AddAttr("alpha", "The scale of Out").SetDefault(1.0f); AddAttr( "use_mkldnn", "(bool, default false) Indicates if MKL-DNN kernel will be used") diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index 4ac7b663418cea..8dd0d11e38d17c 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -424,7 +424,7 @@ class MatMulMKLDNNHandler dnnl::primitive_attr CreatePostOps(const ExecutionContext& ctx) { dnnl::primitive_attr matmul_attrs; dnnl::post_ops post_operations; - + float scale_out = ComputeOutputScale(ctx); if (scale_out != 1.0f) { matmul_attrs.set_output_scales(0, {scale_out}); From c370d2c5c4a7f40b98768d3e77b9fddc20517fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Mon, 6 Jun 2022 04:56:10 -0700 Subject: [PATCH 03/27] gpd --- .../framework/ir/graph_pattern_detector.cc | 17 +++++++++++++++++ .../fluid/framework/ir/graph_pattern_detector.h | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index ea101125b18d2d..9347799ab79d61 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -955,6 +955,23 @@ PDNode *patterns::ElementwiseActivation::operator()( return activation_out_var; } +PDNode *patterns::MatmulActivation::operator()( + const std::string &matmul_type, const std::string &activation_type) { + auto *matmul_op = pattern->NewNode(matmul_repr())->assert_is_op(matmul_type); + auto *matmul_out_var = pattern->NewNode(matmul_out_repr()) + ->AsIntermediate() + ->assert_is_only_output_of_op(matmul_type) + ->assert_is_op_input(activation_type); + auto *activation_op = + pattern->NewNode(activation_repr())->assert_is_op(activation_type); + auto *activation_out_var = pattern->NewNode(activation_out_repr()) + ->AsOutput() + ->assert_is_op_output(activation_type); + matmul_op->LinksTo({matmul_out_var}); + activation_op->LinksFrom({matmul_out_var}).LinksTo({activation_out_var}); + return activation_out_var; +} + PDNode *patterns::SeqConvEltAddRelu::operator()( paddle::framework::ir::PDNode *seqconv_input) { // Create Operators diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index 3c6b6ce94e23f3..09e8f8f6431906 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -549,6 +549,20 @@ struct ElementwiseActivation : public PatternBase { PATTERN_DECL_NODE(activation_out); }; +// Matmul with Activation +struct MatmulActivation : public PatternBase { + MatmulActivation(PDPattern* pattern, const std::string& name_scope) + : PatternBase(pattern, name_scope, "matmul_activation") {} + + PDNode* operator()(const std::string& matmul_type, + const std::string& activation_type); + + PATTERN_DECL_NODE(matmul_op); + PATTERN_DECL_NODE(matmul_out); + PATTERN_DECL_NODE(activation); + PATTERN_DECL_NODE(activation_out); +}; + // SEQCONV with Elementwise_Add ReLU // op: seqconv + elementwise_add + relu // named nodes: From acd609ebf818b8c04cbe03ea67686fc80d3667e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Thu, 9 Jun 2022 02:17:30 -0700 Subject: [PATCH 04/27] format style --- paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index affa66427ac6ea..12867a482c79fe 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -13,19 +13,21 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/mkldnn/matmul_mkldnn_op.h" + #include + #include "paddle/fluid/framework/convert_utils.h" using dnnl::memory; using dnnl::primitive; using paddle::framework::DataLayout; using paddle::framework::ExecutionContext; -using phi::vectorize; using paddle::platform::GetMKLDNNFormat; -using paddle::platform::MKLDNNFormatForSize; using paddle::platform::MKLDNNDeviceContext; +using paddle::platform::MKLDNNFormatForSize; using paddle::platform::MKLDNNGetDataType; using paddle::platform::to_void_cast; +using phi::vectorize; using Tensor = paddle::framework::Tensor; namespace { @@ -421,7 +423,7 @@ class MatMulMKLDNNHandler return std::make_tuple(x_offset_, y_offset_, out_offset_); } -dnnl::primitive_attr CreateMatmulAttrs(const ExecutionContext& ctx) { + dnnl::primitive_attr CreateMatmulAttrs(const ExecutionContext& ctx) { dnnl::primitive_attr matmul_attrs; dnnl::post_ops post_operations; From 8889c29c0ddfe109eac8d05196d5356f2923afb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Thu, 9 Jun 2022 03:35:17 -0700 Subject: [PATCH 05/27] add matmul+act test --- .../framework/ir/graph_pattern_detector.cc | 4 +- .../framework/ir/graph_pattern_detector.h | 2 +- ...test_mkldnn_matmul_activation_fuse_pass.py | 96 +++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index fccb6574554a3a..bd2153ebcce5c1 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -959,13 +959,13 @@ PDNode *patterns::ElementwiseActivation::operator()( PDNode *patterns::MatmulActivation::operator()( const std::string &matmul_type, const std::string &activation_type) { - auto *matmul_op = pattern->NewNode(matmul_repr())->assert_is_op(matmul_type); + auto *matmul_op = pattern->NewNode(matmul_op_repr())->assert_is_op(matmul_type); auto *matmul_out_var = pattern->NewNode(matmul_out_repr()) ->AsIntermediate() ->assert_is_only_output_of_op(matmul_type) ->assert_is_op_input(activation_type); auto *activation_op = - pattern->NewNode(activation_repr())->assert_is_op(activation_type); + pattern->NewNode(activation_op_repr())->assert_is_op(activation_type); auto *activation_out_var = pattern->NewNode(activation_out_repr()) ->AsOutput() ->assert_is_op_output(activation_type); diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index 09e8f8f6431906..50ba8bf8228e0f 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -559,7 +559,7 @@ struct MatmulActivation : public PatternBase { PATTERN_DECL_NODE(matmul_op); PATTERN_DECL_NODE(matmul_out); - PATTERN_DECL_NODE(activation); + PATTERN_DECL_NODE(activation_op); PATTERN_DECL_NODE(activation_out); }; diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py new file mode 100644 index 00000000000000..2c9f1435c42961 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -0,0 +1,96 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from auto_scan_test import PassAutoScanTest +from program_config import TensorConfig, ProgramConfig, OpConfig +import numpy as np +from functools import partial +import unittest +import hypothesis.strategies as st + + +class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): + + def sample_program_config(self, draw): + transpose_X = draw(st.booleans()) + transpose_Y = draw(st.booleans()) + alpha = draw(st.sampled_from([0.5, 1, 2])) + batch_size = draw(st.sampled_from([1, 4])) + channel = draw(st.sampled_from([1, 4])) + input_dim = draw(st.sampled_from([32, 64])) + activation_type = draw(st.sampled_from( + ['relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish'])) + + def generate_input(type): + if transpose_X and transpose_Y: + shape_x = [batch_size, channel, input_dim, 32] + shape_y = [batch_size, channel, 64, input_dim] + elif transpose_X: + shape_x = [batch_size, channel, input_dim, 32] + shape_y = [batch_size, channel, input_dim, 64] + elif transpose_Y: + shape_x = [batch_size, channel, 32, input_dim] + shape_y = [batch_size, channel, 8, input_dim] + else: + shape_x = [batch_size, channel, 32, input_dim] + shape_y = [batch_size, channel, input_dim, 16] + + if type == 'x': + return np.random.random(shape_x).astype(np.float32) + else: + return np.random.random(shape_y).astype(np.float32) + + matmul_op = OpConfig(type='matmul', + inputs={ + 'X': ['matmul_X'], + 'Y': ['matmul_Y'] + }, + outputs={'Out': ['matmul_output']}, + attrs={ + 'transpose_X': transpose_X, + 'transpose_Y': transpose_Y, + 'alpha': alpha + }) + + activation_op = OpConfig(type=activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + attrs={}) + + model_net = [matmul_op, activation_op] + + program_config = ProgramConfig( + ops=model_net, + weights={}, + inputs={ + 'matmul_X': + TensorConfig(data_gen=partial(generate_input, 'x')), + 'matmul_Y': + TensorConfig(data_gen=partial(generate_input, 'y')) + }, + outputs=['activation_output']) + + return program_config + + def sample_predictor_configs(self, program_config): + config = self.create_inference_config(use_mkldnn=True) + yield config, ['matmul'], (1e-5, 1e-5) + + def test(self): + self.run_and_statis(quant=False, + passes=['matmul_activation_mkldnn_fuse_pass']) + + +if __name__ == '__main__': + unittest.main() From d055527a3455c5547d004af9add103a7fcecb9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 14 Jun 2022 03:58:27 -0700 Subject: [PATCH 06/27] implement matmul+activation --- paddle/fluid/framework/ir/CMakeLists.txt | 1 + .../matmul_activation_mkldnn_fuse_pass.cc | 288 ++++++++++++++++++ .../matmul_activation_mkldnn_fuse_pass.h | 41 +++ .../inference/api/paddle_pass_builder.cc | 1 + .../operators/mkldnn/matmul_mkldnn_op.cc | 27 ++ paddle/fluid/platform/mkldnn_reuse.h | 27 ++ ...test_mkldnn_matmul_activation_fuse_pass.py | 42 ++- 7 files changed, 417 insertions(+), 10 deletions(-) create mode 100644 paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc create mode 100644 paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index 374b5490d5da15..e7292d3fe28ceb 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -208,6 +208,7 @@ if(WITH_MKLDNN) pass_library(shuffle_channel_mkldnn_detect_pass inference DIR mkldnn) pass_library(fc_act_mkldnn_fuse_pass inference DIR mkldnn) pass_library(elt_act_mkldnn_fuse_pass inference DIR mkldnn) + pass_library(matmul_activation_mkldnn_fuse_pass inference DIR mkldnn) pass_library(cpu_quantize_placement_pass base DIR mkldnn) pass_library(cpu_quantize_pass inference DIR mkldnn) pass_library(cpu_quantize_squash_pass inference DIR mkldnn) diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc new file mode 100644 index 00000000000000..1f2f6d31bbe92d --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -0,0 +1,288 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h" + +#include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/string/pretty_log.h" + +namespace paddle { +namespace framework { +namespace ir { + +using string::PrettyLogDetail; + +void MatmulActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { + std::vector act_types = { + "relu", "mish", "swish", "sqrt", "hard_swish", "sigmoid", "abs", + "gelu", "relu6", "clip", "tanh", "hard_sigmoid", "leaky_relu"}; + + std::vector matmul_types = {"matmul"}; + + for (const auto& matmul_type : matmul_types) + for (auto& act_type : act_types) { + std::unordered_map attrs_map; + + if (act_type == "swish") + attrs_map.emplace("beta", "activation_alpha"); + else if (act_type == "relu6") + attrs_map.emplace("threshold", "activation_alpha"); + else if (act_type == "hard_sigmoid") { + attrs_map.emplace("slope", "activation_alpha"); + attrs_map.emplace("offset", "activation_beta"); + } else if (act_type == "clip") { + attrs_map.emplace("min", "activation_alpha"); + attrs_map.emplace("max", "activation_beta"); + } else { + attrs_map.emplace("alpha", "activation_alpha"); + attrs_map.emplace("beta", "activation_beta"); + } + FuseMatmulAct(graph, matmul_type, act_type, attrs_map); + } +} + +void MatmulActivationMkldnnFusePass::FuseMatmulAct( + Graph* graph, const std::string& matmul_type, std::string& act_type, + const std::unordered_map& attrs_map) const { + PADDLE_ENFORCE_NOT_NULL( + graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); + FusePassBase::Init(matmul_type + "_" + act_type + "_mkldnn_fuse_pass", graph); + + GraphPatternDetector gpd; + patterns::MatmulActivation matmul_act_pattern( + gpd.mutable_pattern(), "matmul_activation_mkldnn_fuse"); + matmul_act_pattern(matmul_type, act_type); + + int found_matmul_activation_count = 0; + auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, + Graph* g) { + VLOG(4) << "handle " + matmul_type + "+" + act_type + " fuse"; + + if (!IsCompat(subgraph, g)) { + LOG(WARNING) << "matmul_activation_mkldnn_fuse_pass op compat failed."; + return; + } + + GET_IR_NODE_FROM_SUBGRAPH(matmul, matmul_op, matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(matmul_out, matmul_out, matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(activation, activation_op, matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, + matmul_act_pattern); + + OpDesc* matmul_op = matmul->Op(); + OpDesc* act_op = activation->Op(); + + for (const auto& attrs : attrs_map) { + if (act_op->HasAttr(attrs.first)) { + matmul_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); + } + } + + if (act_type == "gelu" && activation->Op()->HasAttr("approximate")) { + act_type = BOOST_GET_CONST(bool, activation->Op()->GetAttr("approximate")) + ? "gelu_tanh" + : "gelu_erf"; + matmul_op->SetAttr("activation_alpha", 0.0f); + matmul_op->SetAttr("activation_beta", 0.0f); + } + matmul_op->SetAttr("activation_type", act_type); + matmul_op->SetOutput("Out", {activation_out->Name()}); + + IR_NODE_LINK_TO(matmul, activation_out); + GraphSafeRemoveNodes(graph, {activation, matmul_out}); + found_matmul_activation_count++; + }; + + gpd(graph, handler); + AddStatis(found_matmul_activation_count); + if (!Has("disable_logs") || !Get("disable_logs")) { + PrettyLogDetail("--- fused %d matmul with %s activation", + found_matmul_activation_count, act_type); + } +} + +MatmulActivationMkldnnFusePass::MatmulActivationMkldnnFusePass() { + AddOpCompat(OpCompat("matmul")) + .AddInput("X") + .IsTensor() + .End() + .AddInput("Y") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("alpha") + .IsType() + .End() + .AddAttr("transpose_X") + .IsType() + .End() + .AddAttr("transpose_Y") + .IsType() + .End(); + + AddOpCompat(OpCompat("relu")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End(); + + AddOpCompat(OpCompat("leaky_relu")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("alpha") + .IsType() + .End(); + + AddOpCompat(OpCompat("relu6")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("threshold") + .IsType() + .End(); + + AddOpCompat(OpCompat("swish")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("beta") + .IsType() + .End(); + + AddOpCompat(OpCompat("hard_swish")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("threshold") + .IsOptional() + .IsType() + .End() + .AddAttr("scale") + .IsOptional() + .IsType() + .End() + .AddAttr("offset") + .IsOptional() + .IsType() + .End(); + + AddOpCompat(OpCompat("mish")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End(); + + AddOpCompat(OpCompat("hard_sigmoid")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("slope") + .IsOptional() + .IsType() + .End() + .AddAttr("offset") + .IsOptional() + .IsType() + .End(); + + AddOpCompat(OpCompat("gelu")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("approximate") + .IsType() + .End(); + + AddOpCompat(OpCompat("tanh")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End(); + + AddOpCompat(OpCompat("sigmoid")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End(); + + AddOpCompat(OpCompat("sqrt")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End(); + + AddOpCompat(OpCompat("abs")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End(); +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +REGISTER_PASS(matmul_activation_mkldnn_fuse_pass, + paddle::framework::ir::MatmulActivationMkldnnFusePass); + +REGISTER_PASS_CAPABILITY(matmul_activation_mkldnn_fuse_pass) + .AddCombination( + paddle::framework::compatible::OpVersionComparatorCombination() + .LE("matmul", 1) + .EQ("abs", 0) + .LE("clip", 1) + .EQ("gelu", 0) + .EQ("hard_sigmoid", 0) + .LE("hard_swish", 0) + .LE("leaky_relu", 1) + .LE("mish", 1) + .EQ("relu", 0) + .EQ("relu6", 0) + .EQ("sigmoid", 0) + .EQ("sqrt", 0) + .EQ("swish", 0) + .EQ("tanh", 0)); \ No newline at end of file diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h new file mode 100644 index 00000000000000..88ec5072220625 --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h @@ -0,0 +1,41 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph.h" + +namespace paddle { +namespace framework { +namespace ir { + +class MatmulActivationMkldnnFusePass : public FusePassBase { + public: + MatmulActivationMkldnnFusePass(); + virtual ~MatmulActivationMkldnnFusePass() {} + + protected: + void ApplyImpl(Graph *graph) const override; + + void FuseMatmulAct( + Graph *graph, const std::string &matmul_type, std::string &act_type, + const std::unordered_map &attrs_map) const; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle \ No newline at end of file diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc index 9e5b76db4ac161..ccf7283cb06aa1 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.cc +++ b/paddle/fluid/inference/api/paddle_pass_builder.cc @@ -320,6 +320,7 @@ void CpuPassStrategy::EnableMKLDNN() { "softplus_activation_mkldnn_fuse_pass", // "shuffle_channel_mkldnn_detect_pass", // "elt_act_mkldnn_fuse_pass", // + "matmul_activation_mkldnn_fuse_pass", // // TODO(intel): Please fix the bug on windows. // https://github.com/PaddlePaddle/Paddle/issues/29710 // "mkldnn_inplace_pass", // This pass should be activated after diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index 12867a482c79fe..88ce8518c0e978 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/mkldnn/matmul_mkldnn_op.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" #include @@ -432,6 +433,32 @@ class MatMulMKLDNNHandler matmul_attrs.set_output_scales(0, {scale_out}); } + if (ctx.HasAttr("activation_type")) { + const auto activation_type = ctx.Attr("activation_type"); + const float activation_scale = ctx.HasAttr("activation_scale") + ? ctx.Attr("activation_scale") + : 1.0f; + const float alpha = ctx.HasAttr("activation_alpha") + ? ctx.Attr("activation_alpha") + : 0.0f; + const float beta = ctx.HasAttr("activation_beta") + ? ctx.Attr("activation_beta") + : 0.0f; + + if (activation_type == "hard_sigmoid") { + post_operations.append_eltwise(activation_scale, + dnnl::algorithm::eltwise_linear, + alpha, beta); + post_operations.append_eltwise( + activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); + } else if (activation_type != "") { + const auto activation_algorithm = + paddle::platform::AcquireActivationAlgorithm(activation_type); + post_operations.append_eltwise(activation_scale, activation_algorithm, + alpha, beta); + } + } + matmul_attrs.set_post_ops(post_operations); return matmul_attrs; } diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index 382f96e83bfce5..48a11c8de31942 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -951,6 +951,33 @@ class ActivationMKLDNNHandler } }; +static const dnnl::algorithm AcquireActivationAlgorithm( + std::string activation_name) { + std::unordered_map activation_map = { + {"abs", dnnl::algorithm::eltwise_abs}, + {"clip", dnnl::algorithm::eltwise_clip}, + {"gelu", dnnl::algorithm::eltwise_gelu_erf}, + {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf}, + {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh}, + {"hard_swish", dnnl::algorithm::eltwise_hardswish}, + {"leaky_relu", dnnl::algorithm::eltwise_relu}, + {"mish", dnnl::algorithm::eltwise_mish}, + {"relu", dnnl::algorithm::eltwise_relu}, + {"relu6", dnnl::algorithm::eltwise_bounded_relu}, + {"sigmoid", dnnl::algorithm::eltwise_logistic}, + {"sqrt", dnnl::algorithm::eltwise_sqrt}, + {"swish", dnnl::algorithm::eltwise_swish}, + {"tanh", dnnl::algorithm::eltwise_tanh}}; + + const auto& activation_type = activation_map.find(activation_name); + + PADDLE_ENFORCE_NE(activation_type, activation_map.end(), + platform::errors::InvalidArgument( + "Activation '%s' not found in oneDNN algorithms mapper", + activation_name)); + return activation_type->second; +} + class ReorderMKLDNNHandler { public: ReorderMKLDNNHandler(std::vector& dims, // NOLINT diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index 2c9f1435c42961..3b1307a660e14f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -25,12 +25,12 @@ class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): def sample_program_config(self, draw): transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) - alpha = draw(st.sampled_from([0.5, 1, 2])) - batch_size = draw(st.sampled_from([1, 4])) - channel = draw(st.sampled_from([1, 4])) - input_dim = draw(st.sampled_from([32, 64])) - activation_type = draw(st.sampled_from( - ['relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish'])) + alpha = draw(st.sampled_from([1, 2])) + batch_size = draw(st.sampled_from([4])) + channel = draw(st.sampled_from([8])) + input_dim = draw(st.sampled_from([32])) + activation_type = draw(st.sampled_from(['relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', + 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', 'leaky_relu'])) def generate_input(type): if transpose_X and transpose_Y: @@ -63,10 +63,32 @@ def generate_input(type): 'alpha': alpha }) - activation_op = OpConfig(type=activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - attrs={}) + if activation_type == "relu6": + activation_op = OpConfig(activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0))) + elif activation_type == "leaky_relu": + activation_op = OpConfig(activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0))) + elif activation_type == "swish": + activation_op = OpConfig(activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=1.0))) + elif activation_type == "clip": + activation_op = OpConfig(activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + min=draw( + st.floats(min_value=0.1, max_value=0.49)), + max=draw(st.floats(min_value=0.5, max_value=1.0))) + else: + activation_op = OpConfig(activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}) model_net = [matmul_op, activation_op] From a518f9486a352270bea5393aff26e9209d6fbbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 14 Jun 2022 04:07:33 -0700 Subject: [PATCH 07/27] whitespaces --- .../framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc | 2 +- .../framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index 1f2f6d31bbe92d..a2e8e1266fea1d 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -285,4 +285,4 @@ REGISTER_PASS_CAPABILITY(matmul_activation_mkldnn_fuse_pass) .EQ("sigmoid", 0) .EQ("sqrt", 0) .EQ("swish", 0) - .EQ("tanh", 0)); \ No newline at end of file + .EQ("tanh", 0)); diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h index 88ec5072220625..8575354d9d19cf 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h @@ -38,4 +38,4 @@ class MatmulActivationMkldnnFusePass : public FusePassBase { } // namespace ir } // namespace framework -} // namespace paddle \ No newline at end of file +} // namespace paddle From 18eef24425a43358598eda3a805a9a6b43600cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 14 Jun 2022 05:02:11 -0700 Subject: [PATCH 08/27] code style --- .../framework/ir/graph_pattern_detector.cc | 3 +- .../operators/mkldnn/matmul_mkldnn_op.cc | 13 +++---- paddle/fluid/platform/mkldnn_reuse.h | 2 +- ...test_mkldnn_matmul_activation_fuse_pass.py | 38 +++++++++++-------- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index bd2153ebcce5c1..ff3d51ed91d9e7 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -959,7 +959,8 @@ PDNode *patterns::ElementwiseActivation::operator()( PDNode *patterns::MatmulActivation::operator()( const std::string &matmul_type, const std::string &activation_type) { - auto *matmul_op = pattern->NewNode(matmul_op_repr())->assert_is_op(matmul_type); + auto *matmul_op = + pattern->NewNode(matmul_op_repr())->assert_is_op(matmul_type); auto *matmul_out_var = pattern->NewNode(matmul_out_repr()) ->AsIntermediate() ->assert_is_only_output_of_op(matmul_type) diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index 88ce8518c0e978..a856ec3893ffb0 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -13,11 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/mkldnn/matmul_mkldnn_op.h" -#include "paddle/fluid/platform/mkldnn_reuse.h" #include #include "paddle/fluid/framework/convert_utils.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" using dnnl::memory; using dnnl::primitive; @@ -434,10 +434,10 @@ class MatMulMKLDNNHandler } if (ctx.HasAttr("activation_type")) { - const auto activation_type = ctx.Attr("activation_type"); + const auto activation_type = ctx.Attr("activation_type"); const float activation_scale = ctx.HasAttr("activation_scale") - ? ctx.Attr("activation_scale") - : 1.0f; + ? ctx.Attr("activation_scale") + : 1.0f; const float alpha = ctx.HasAttr("activation_alpha") ? ctx.Attr("activation_alpha") : 0.0f; @@ -446,9 +446,8 @@ class MatMulMKLDNNHandler : 0.0f; if (activation_type == "hard_sigmoid") { - post_operations.append_eltwise(activation_scale, - dnnl::algorithm::eltwise_linear, - alpha, beta); + post_operations.append_eltwise( + activation_scale, dnnl::algorithm::eltwise_linear, alpha, beta); post_operations.append_eltwise( activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); } else if (activation_type != "") { diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index 48a11c8de31942..3e0fabaac25c8d 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -974,7 +974,7 @@ static const dnnl::algorithm AcquireActivationAlgorithm( PADDLE_ENFORCE_NE(activation_type, activation_map.end(), platform::errors::InvalidArgument( "Activation '%s' not found in oneDNN algorithms mapper", - activation_name)); + activation_name)); return activation_type->second; } diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index 3b1307a660e14f..6eed0c88d9cca4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -29,8 +29,12 @@ def sample_program_config(self, draw): batch_size = draw(st.sampled_from([4])) channel = draw(st.sampled_from([8])) input_dim = draw(st.sampled_from([32])) - activation_type = draw(st.sampled_from(['relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', - 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', 'leaky_relu'])) + activation_type = draw( + st.sampled_from([ + 'relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', + 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', + 'hard_sigmoid', 'leaky_relu' + ])) def generate_input(type): if transpose_X and transpose_Y: @@ -67,24 +71,30 @@ def generate_input(type): activation_op = OpConfig(activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, - threshold=draw(st.floats(min_value=1.0, max_value=10.0))) + threshold=draw( + st.floats(min_value=1.0, + max_value=10.0))) elif activation_type == "leaky_relu": activation_op = OpConfig(activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, - alpha=draw(st.floats(min_value=0.1, max_value=1.0))) + alpha=draw( + st.floats(min_value=0.1, + max_value=1.0))) elif activation_type == "swish": activation_op = OpConfig(activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, - beta=draw(st.floats(min_value=0.1, max_value=1.0))) + beta=draw( + st.floats(min_value=0.1, + max_value=1.0))) elif activation_type == "clip": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - min=draw( - st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + min=draw(st.floats(min_value=0.1, max_value=0.49)), + max=draw(st.floats(min_value=0.5, max_value=1.0))) else: activation_op = OpConfig(activation_type, inputs={"X": ["matmul_output"]}, @@ -96,10 +106,8 @@ def generate_input(type): ops=model_net, weights={}, inputs={ - 'matmul_X': - TensorConfig(data_gen=partial(generate_input, 'x')), - 'matmul_Y': - TensorConfig(data_gen=partial(generate_input, 'y')) + 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'x')), + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'y')) }, outputs=['activation_output']) From 695a5708463a8feb641a688b044b5631aae53f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 14 Jun 2022 05:33:02 -0700 Subject: [PATCH 09/27] python code format --- .../test_mkldnn_matmul_activation_fuse_pass.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index 6eed0c88d9cca4..d3167c51bc66fc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -72,22 +72,22 @@ def generate_input(type): inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + st.floats(min_value=1.0, + max_value=10.0))) elif activation_type == "leaky_relu": activation_op = OpConfig(activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + st.floats(min_value=0.1, + max_value=1.0))) elif activation_type == "swish": activation_op = OpConfig(activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + st.floats(min_value=0.1, + max_value=1.0))) elif activation_type == "clip": activation_op = OpConfig( activation_type, From d7ac6984e38609f6feda175a43bbb718dfbaf8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 14 Jun 2022 10:56:29 -0700 Subject: [PATCH 10/27] Increase UT timeout --- python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt | 1 + .../ir/inference/test_mkldnn_matmul_activation_fuse_pass.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt index 3687d09653fd83..368b27da5554ce 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt @@ -204,5 +204,6 @@ if(WITH_GPU AND TENSORRT_FOUND) PROPERTIES TIMEOUT 120) set_tests_properties(test_mkldnn_conv_affine_channel_fuse_pass PROPERTIES TIMEOUT 60) + set_tests_properties(test_mkldnn_matmul_activation_fuse_pass PROPERTIES TIMEOUT 300) endif() endif() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index d3167c51bc66fc..259112d43cb3b5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -119,6 +119,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, + max_examples=300, passes=['matmul_activation_mkldnn_fuse_pass']) From e0de057ef5c45fb14af8f8520515044c6095b8e9 Mon Sep 17 00:00:00 2001 From: Silv3S Date: Tue, 14 Jun 2022 20:30:41 +0200 Subject: [PATCH 11/27] code format --- .../paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt index 368b27da5554ce..1d3ea1e3ed8f37 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt @@ -204,6 +204,7 @@ if(WITH_GPU AND TENSORRT_FOUND) PROPERTIES TIMEOUT 120) set_tests_properties(test_mkldnn_conv_affine_channel_fuse_pass PROPERTIES TIMEOUT 60) - set_tests_properties(test_mkldnn_matmul_activation_fuse_pass PROPERTIES TIMEOUT 300) + set_tests_properties(test_mkldnn_matmul_activation_fuse_pass + PROPERTIES TIMEOUT 300) endif() endif() From 008bf1b5a892e19605a2e1168d28fcb34403b1e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 28 Jun 2022 03:19:41 -0700 Subject: [PATCH 12/27] update style --- .../matmul_activation_mkldnn_fuse_pass.cc | 27 ++++++++++++++----- .../matmul_activation_mkldnn_fuse_pass.h | 4 ++- .../operators/mkldnn/matmul_mkldnn_op.cc | 4 +-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index a2e8e1266fea1d..901e305ff90d04 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -24,9 +24,19 @@ namespace ir { using string::PrettyLogDetail; void MatmulActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - std::vector act_types = { - "relu", "mish", "swish", "sqrt", "hard_swish", "sigmoid", "abs", - "gelu", "relu6", "clip", "tanh", "hard_sigmoid", "leaky_relu"}; + std::vector act_types = {"abs", + "clip", + "gelu", + "hard_sigmoid", + "hard_swish", + "leaky_relu", + "mish", + "relu", + "relu6", + "sigmoid", + "sqrt", + "swish", + "tanh"}; std::vector matmul_types = {"matmul"}; @@ -53,7 +63,9 @@ void MatmulActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { } void MatmulActivationMkldnnFusePass::FuseMatmulAct( - Graph* graph, const std::string& matmul_type, std::string& act_type, + Graph* graph, + const std::string& matmul_type, + std::string& act_type, const std::unordered_map& attrs_map) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); @@ -77,8 +89,8 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( GET_IR_NODE_FROM_SUBGRAPH(matmul, matmul_op, matmul_act_pattern); GET_IR_NODE_FROM_SUBGRAPH(matmul_out, matmul_out, matmul_act_pattern); GET_IR_NODE_FROM_SUBGRAPH(activation, activation_op, matmul_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, - matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH( + activation_out, activation_out, matmul_act_pattern); OpDesc* matmul_op = matmul->Op(); OpDesc* act_op = activation->Op(); @@ -108,7 +120,8 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( AddStatis(found_matmul_activation_count); if (!Has("disable_logs") || !Get("disable_logs")) { PrettyLogDetail("--- fused %d matmul with %s activation", - found_matmul_activation_count, act_type); + found_matmul_activation_count, + act_type); } } diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h index 8575354d9d19cf..0e8d613e8ae9d1 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h @@ -32,7 +32,9 @@ class MatmulActivationMkldnnFusePass : public FusePassBase { void ApplyImpl(Graph *graph) const override; void FuseMatmulAct( - Graph *graph, const std::string &matmul_type, std::string &act_type, + Graph *graph, + const std::string &matmul_type, + std::string &act_type, const std::unordered_map &attrs_map) const; }; diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index caf50cfc2c3c76..a021d1b88f7af2 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -474,8 +474,8 @@ class MatMulMKLDNNHandler } else if (activation_type != "") { const auto activation_algorithm = paddle::platform::AcquireActivationAlgorithm(activation_type); - post_operations.append_eltwise(activation_scale, activation_algorithm, - alpha, beta); + post_operations.append_eltwise( + activation_scale, activation_algorithm, alpha, beta); } } From 48827ec553b6b23620aa5d030cc5366950170aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 05:16:27 -0700 Subject: [PATCH 13/27] generalize activation fuse passes --- paddle/fluid/framework/ir/CMakeLists.txt | 6 +- .../framework/ir/graph_pattern_detector.cc | 92 +---- .../framework/ir/graph_pattern_detector.h | 64 +--- .../conv_activation_mkldnn_fuse_pass.cc | 324 ------------------ ...conv_activation_mkldnn_fuse_pass_tester.cc | 7 +- .../ir/mkldnn/elt_act_mkldnn_fuse_pass.cc | 159 --------- .../ir/mkldnn/elt_act_mkldnn_fuse_pass.h | 46 --- .../operator_activation_mkldnn_fuse_pass.cc | 174 ++++++++++ ...=> operator_activation_mkldnn_fuse_pass.h} | 13 +- .../softplus_activation_mkldnn_fuse_pass.cc | 145 -------- .../softplus_activation_mkldnn_fuse_pass.h | 45 --- ...plus_activation_mkldnn_fuse_pass_tester.cc | 6 +- .../inference/api/paddle_pass_builder.cc | 14 +- paddle/fluid/operators/activation_op.cc | 20 -- .../mkldnn/elementwise_mkldnn_op.h | 37 +- .../operators/mkldnn/softplus_mkldnn_op.h | 57 ++- .../quantization/quant2_int8_mkldnn_pass.py | 3 +- .../test_conv_act_mkldnn_fuse_pass.py | 2 +- .../test_mkldnn_conv_activation_fuse_pass.py | 2 +- .../test_mkldnn_conv_gelu_fuse_pass.py | 2 +- ...test_mkldnn_conv_hard_sigmoid_fuse_pass.py | 2 +- .../test_mkldnn_conv_hard_swish_fuse_pass.py | 2 +- .../test_mkldnn_conv_mish_fuse_pass.py | 2 +- .../test_mkldnn_elt_act_fuse_pass.py | 2 +- .../test_mkldnn_elt_act_fuse_pass_new.py | 2 +- ...st_mkldnn_softplus_activation_fuse_pass.py | 222 +++++------- 26 files changed, 356 insertions(+), 1094 deletions(-) delete mode 100644 paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc delete mode 100644 paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc delete mode 100644 paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h create mode 100644 paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc rename paddle/fluid/framework/ir/mkldnn/{conv_activation_mkldnn_fuse_pass.h => operator_activation_mkldnn_fuse_pass.h} (81%) delete mode 100644 paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc delete mode 100644 paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index d19b163817e41e..b9d6ba58d6bc37 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -197,7 +197,7 @@ if(WITH_MKLDNN) pass_library(depthwise_conv_mkldnn_pass base DIR mkldnn) pass_library(conv_affine_channel_mkldnn_fuse_pass inference DIR mkldnn) pass_library(conv_bias_mkldnn_fuse_pass inference DIR mkldnn) - pass_library(conv_activation_mkldnn_fuse_pass inference DIR mkldnn) + pass_library(operator_activation_mkldnn_fuse_pass inference DIR mkldnn) pass_library(conv_concat_relu_mkldnn_fuse_pass inference DIR mkldnn) pass_library(conv_elementwise_add_mkldnn_fuse_pass inference DIR mkldnn) pass_library(int8_scale_calculation_mkldnn_pass inference DIR mkldnn) @@ -208,10 +208,8 @@ if(WITH_MKLDNN) pass_library(cpu_bfloat16_pass inference DIR mkldnn) pass_library(fc_mkldnn_pass inference DIR mkldnn) pass_library(interpolate_mkldnn_pass inference DIR mkldnn) - pass_library(softplus_activation_mkldnn_fuse_pass inference DIR mkldnn) pass_library(shuffle_channel_mkldnn_detect_pass inference DIR mkldnn) pass_library(fc_act_mkldnn_fuse_pass inference DIR mkldnn) - pass_library(elt_act_mkldnn_fuse_pass inference DIR mkldnn) pass_library(cpu_quantize_placement_pass base DIR mkldnn) pass_library(cpu_quantize_pass inference DIR mkldnn) pass_library(cpu_quantize_squash_pass inference DIR mkldnn) @@ -404,7 +402,7 @@ if(WITH_MKLDNN) cc_test( test_conv_activation_mkldnn_fuse_pass SRCS mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc - DEPS conv_activation_mkldnn_fuse_pass) + DEPS operator_activation_mkldnn_fuse_pass) cc_test( test_conv_concat_relu_mkldnn_fuse_pass SRCS mkldnn/conv_concat_relu_mkldnn_fuse_pass_tester.cc diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index d5b61228868504..32aed8e504c6cf 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -925,65 +925,22 @@ PDNode *patterns::ConvBN::operator()(paddle::framework::ir::PDNode *conv_input, return bn_out_var; } -PDNode *patterns::ConvActivation::operator()( - paddle::framework::ir::PDNode *conv_input, - std::string conv_type, - std::string activation_type) { - // Create Operators - conv_input->assert_is_op_input(conv_type, "Input"); - auto *conv_op = pattern->NewNode(conv_repr())->assert_is_op(conv_type); - auto *activation_op = - pattern->NewNode(activation_repr())->assert_is_op(activation_type); - // Create variables - // Filter - auto *conv_weight_var = pattern->NewNode(conv_weight_repr()) - ->AsInput() - ->assert_is_persistable_var() - ->assert_is_op_input(conv_type, "Filter"); - // intermediate variable, will be removed in the IR after fuse. - auto *conv_out_var = pattern->NewNode(conv_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op(conv_type) - ->assert_is_op_input(activation_type); - // output - auto *activation_out_var = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - - conv_op->LinksFrom({conv_input, conv_weight_var}).LinksTo({conv_out_var}); - activation_op->LinksFrom({conv_out_var}).LinksTo({activation_out_var}); - return activation_out_var; -} - -PDNode *patterns::ElementwiseActivation::operator()( - paddle::framework::ir::PDNode *elementwise_a, - const std::string &elementwise_type, - const std::string &activation_type) { - // Create Operators - elementwise_a->assert_is_op_input(elementwise_type, "X"); - auto *elementwise_op = - pattern->NewNode(elementwise_repr())->assert_is_op(elementwise_type); +PDNode *patterns::OperatorActivation::operator()( + const std::string &operator_type, const std::string &activation_type) { + auto *preceding_op = + pattern->NewNode(preceding_op_repr())->assert_is_op(operator_type); + auto *preceding_op_out = pattern->NewNode(preceding_op_out_repr()) + ->AsIntermediate() + ->assert_is_only_output_of_op(operator_type) + ->assert_is_op_input(activation_type); auto *activation_op = pattern->NewNode(activation_repr())->assert_is_op(activation_type); - // Create variables - auto *elementwise_b = pattern->NewNode(elementwise_b_repr()) - ->AsInput() - ->assert_is_op_input(elementwise_type, "Y"); - // intermediate variable, will be removed in the IR after fuse. - auto *elementwise_out_var = - pattern->NewNode(elementwise_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op(elementwise_type) - ->assert_is_op_input(activation_type); - // output - auto *activation_out_var = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - - elementwise_op->LinksFrom({elementwise_a, elementwise_b}) - .LinksTo({elementwise_out_var}); - activation_op->LinksFrom({elementwise_out_var}).LinksTo({activation_out_var}); - return activation_out_var; + auto *activation_out = pattern->NewNode(activation_out_repr()) + ->AsOutput() + ->assert_is_op_output(activation_type); + preceding_op->LinksTo({preceding_op_out}); + activation_op->LinksFrom({preceding_op_out}).LinksTo({activation_out}); + return activation_out; } PDNode *patterns::SeqConvEltAddRelu::operator()( @@ -1132,27 +1089,6 @@ PDNode *patterns::FCActOneDNN::operator()(const std::string &act_type) { return act_out; } -PDNode *patterns::SoftplusActivation::operator()(std::string activation_type) { - // Create Operators - auto *softplus_op = - pattern->NewNode(softplus_repr())->assert_is_op("softplus"); - auto *activation_op = - pattern->NewNode(activation_repr())->assert_is_op(activation_type); - // intermediate variable, will be removed in the IR after fuse. - auto *softplus_out = pattern->NewNode(softplus_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op("softplus") - ->assert_is_op_input(activation_type); - // output - auto *activation_out = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - - softplus_op->LinksTo({softplus_out}); - activation_op->LinksFrom({softplus_out}).LinksTo({activation_out}); - return activation_out; -} - PDNode *patterns::Embedding::operator()(PDNode *x) { x->assert_is_op_input("lookup_table", "Ids"); auto *lookup_table_op = diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index 0cc216b6e0de2c..2cda4c2a0593e4 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -516,49 +516,17 @@ struct ConvBN : public PatternBase { PATTERN_DECL_NODE(bn_saved_variance); }; -// Conv with Activation -// op: conv + activation -// named nodes: -// conv_input, conv_weight, -// conv_out, conv, -// activation_out, activation -struct ConvActivation : public PatternBase { - ConvActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "conv_activation") {} +// Generalized Op with Activation +struct OperatorActivation : public PatternBase { + OperatorActivation(PDPattern* pattern, const std::string& name_scope) + : PatternBase(pattern, name_scope, "operator_activation") {} - PDNode* operator()(PDNode* conv_input, - std::string conv_type = "conv2d", - std::string activation_type = "relu"); - - // declare operator node's name - PATTERN_DECL_NODE(conv); - PATTERN_DECL_NODE(activation); - // declare variable node's name - PATTERN_DECL_NODE(conv_weight); - PATTERN_DECL_NODE(conv_out); - PATTERN_DECL_NODE(activation_out); -}; - -// Elementwise with Activation -// op: elementwise + activation -// named nodes: -// elementwise_a, elementwise_b, -// elementwise_out, elementwise, -// activation_out, activation -struct ElementwiseActivation : public PatternBase { - ElementwiseActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "elementwise_add_activation") {} - - PDNode* operator()(PDNode* elementwise_a, - const std::string& elementwise_type, + PDNode* operator()(const std::string& operator_type, const std::string& activation_type); - // declare operator node's name - PATTERN_DECL_NODE(elementwise); + PATTERN_DECL_NODE(preceding_op); + PATTERN_DECL_NODE(preceding_op_out); PATTERN_DECL_NODE(activation); - // declare variable node's name - PATTERN_DECL_NODE(elementwise_b); - PATTERN_DECL_NODE(elementwise_out); PATTERN_DECL_NODE(activation_out); }; @@ -652,24 +620,6 @@ struct FCActOneDNN : public PatternBase { PATTERN_DECL_NODE(act_out); }; -// Fuse softplus with activation -// ops: softplus + activation -// nodes: -// softplus, softplus_out, -// activation, activation_out -struct SoftplusActivation : public PatternBase { - SoftplusActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "softplus_activation") {} - - PDNode* operator()(std::string activation_type); - - // declare operator node's name - PATTERN_DECL_NODE(softplus); - PATTERN_DECL_NODE(activation); - PATTERN_DECL_NODE(softplus_out); - PATTERN_DECL_NODE(activation_out); -}; - // Embedding struct Embedding : public PatternBase { Embedding(PDPattern* pattern, const std::string& name_scope) diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc deleted file mode 100644 index 4eefc2987bcb44..00000000000000 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h" - -#include "paddle/fluid/framework/op_version_registry.h" -#include "paddle/fluid/string/pretty_log.h" - -namespace paddle { -namespace framework { -namespace ir { - -using string::PrettyLogDetail; - -void ConvActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - std::vector act_types = {"relu", - "mish", - "swish", - "sqrt", - "hard_swish", - "sigmoid", - "abs", - "gelu", - "relu6", - "clip", - "tanh", - "hard_sigmoid", - "leaky_relu"}; - - std::vector conv_types = {"conv2d"}; - - for (const auto& conv_type : conv_types) - for (auto& act_type : act_types) { - std::unordered_map attrs_map; - - if (act_type == "swish") - attrs_map.emplace("beta", "fuse_alpha"); - else if (act_type == "relu6") - attrs_map.emplace("threshold", "fuse_alpha"); - else if (act_type == "hard_sigmoid") { - attrs_map.emplace("slope", "fuse_alpha"); - attrs_map.emplace("offset", "fuse_beta"); - } else if (act_type == "clip") { - attrs_map.emplace("min", "fuse_alpha"); - attrs_map.emplace("max", "fuse_beta"); - } else { - attrs_map.emplace("alpha", "fuse_alpha"); - attrs_map.emplace("beta", "fuse_beta"); - } - FuseConvAct(graph, conv_type, act_type, attrs_map); - } -} - -void ConvActivationMkldnnFusePass::FuseConvAct( - Graph* graph, - const std::string& conv_type, - std::string& act_type, - const std::unordered_map& attrs_map) const { - PADDLE_ENFORCE_NOT_NULL( - graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); - FusePassBase::Init(conv_type + "_" + act_type + "_mkldnn_fuse_pass", graph); - - GraphPatternDetector gpd; - auto* conv_input = gpd.mutable_pattern() - ->NewNode("conv_activation_mkldnn_fuse/conv_input") - ->AsInput() - ->assert_is_op_input(conv_type, "Input"); - patterns::ConvActivation conv_act_pattern(gpd.mutable_pattern(), - "conv_activation_mkldnn_fuse"); - conv_act_pattern(conv_input, conv_type, act_type); - - int found_conv_activation_count = 0; - auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, - Graph* g) { - VLOG(4) << "handle " + conv_type + "+" + act_type + " fuse"; - - if (!IsCompat(subgraph, g)) { - LOG(WARNING) << "conv_activation_mkldnn_fuse_pass op compat failed."; - return; - } - - GET_IR_NODE_FROM_SUBGRAPH(conv_weight, conv_weight, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(conv_out, conv_out, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(conv, conv, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation, activation, conv_act_pattern); - - OpDesc* conv_op = conv->Op(); - OpDesc* act_op = activation->Op(); - - for (const auto& attrs : attrs_map) { - if (act_op->HasAttr(attrs.first)) { - conv_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); - } - } - - if (act_type == "gelu" && activation->Op()->HasAttr("approximate")) { - act_type = BOOST_GET_CONST(bool, activation->Op()->GetAttr("approximate")) - ? "gelu_tanh" - : "gelu_erf"; - conv_op->SetAttr("fuse_alpha", 0.0f); - conv_op->SetAttr("fuse_beta", 0.0f); - } - conv_op->SetAttr("fuse_activation", act_type); - conv_op->SetOutput("Output", {activation_out->Name()}); - - IR_NODE_LINK_TO(conv, activation_out); - GraphSafeRemoveNodes(graph, {activation, conv_out}); - found_conv_activation_count++; - }; - - gpd(graph, handler); - AddStatis(found_conv_activation_count); - if (!Has("disable_logs") || !Get("disable_logs")) { - PrettyLogDetail("--- fused %d conv with %s activation", - found_conv_activation_count, - act_type); - } -} - -ConvActivationMkldnnFusePass::ConvActivationMkldnnFusePass() { - AddOpCompat(OpCompat("conv2d")) - .AddInput("Input") - .IsTensor() - .End() - .AddInput("Filter") - .IsTensor() - .End() - .AddInput("Bias") - .IsOptional() - .IsTensor() - .End() - .AddInput("ResidualData") - .IsTensor() - .IsOptional() - .End() - .AddOutput("Output") - .IsTensor() - .End() - .AddAttr("strides") - .IsType>() - .End() - .AddAttr("paddings") - .IsType>() - .End() - .AddAttr("padding_algorithm") - .IsOptional() - .IsStringIn({"EXPLICIT", "SAME", "VALID"}) - .End() - .AddAttr("groups") - .IsNumGE(1) - .End() - .AddAttr("dilations") - .IsType>() - .End() - .AddAttr("data_format") - .IsOptional() - .IsStringIn({"NCHW", "NHWC", "AnyLayout"}) - .End(); - - AddOpCompat(OpCompat("relu")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End(); - - AddOpCompat(OpCompat("leaky_relu")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End() - .AddAttr("alpha") - .IsType() - .End(); - - AddOpCompat(OpCompat("relu6")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End() - .AddAttr("threshold") - .IsType() - .End(); - - AddOpCompat(OpCompat("swish")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End() - .AddAttr("beta") - .IsType() - .End(); - - AddOpCompat(OpCompat("hard_swish")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End() - .AddAttr("threshold") - .IsOptional() - .IsType() - .End() - .AddAttr("scale") - .IsOptional() - .IsType() - .End() - .AddAttr("offset") - .IsOptional() - .IsType() - .End(); - - AddOpCompat(OpCompat("mish")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End(); - - AddOpCompat(OpCompat("hard_sigmoid")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End() - .AddAttr("slope") - .IsOptional() - .IsType() - .End() - .AddAttr("offset") - .IsOptional() - .IsType() - .End(); - - AddOpCompat(OpCompat("gelu")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End() - .AddAttr("approximate") - .IsType() - .End(); - - AddOpCompat(OpCompat("tanh")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End(); - - AddOpCompat(OpCompat("sigmoid")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End(); - - AddOpCompat(OpCompat("sqrt")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End(); - - AddOpCompat(OpCompat("abs")) - .AddInput("X") - .IsTensor() - .End() - .AddOutput("Out") - .IsTensor() - .End(); -} - -} // namespace ir -} // namespace framework -} // namespace paddle - -REGISTER_PASS(conv_activation_mkldnn_fuse_pass, - paddle::framework::ir::ConvActivationMkldnnFusePass); - -REGISTER_PASS_CAPABILITY(conv_activation_mkldnn_fuse_pass) - .AddCombination( - paddle::framework::compatible::OpVersionComparatorCombination() - .LE("conv2d", 1) - .EQ("abs", 0) - .LE("clip", 1) - .EQ("gelu", 0) - .EQ("hard_sigmoid", 0) - .LE("hard_swish", 0) - .LE("leaky_relu", 1) - .LE("mish", 1) - .EQ("relu", 0) - .EQ("relu6", 0) - .EQ("sigmoid", 0) - .EQ("sqrt", 0) - .EQ("swish", 0) - .EQ("tanh", 0)); diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc index d75874c6de7120..05e6d8a058f24a 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc @@ -16,7 +16,7 @@ #include -#include "paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h" +#include "paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h" #include "paddle/fluid/framework/op_proto_maker.h" namespace paddle { @@ -147,7 +147,8 @@ void MainTest(std::string activation) { std::unique_ptr graph(new ir::Graph(prog)); - auto pass = PassRegistry::Instance().Get("conv_activation_mkldnn_fuse_pass"); + auto pass = + PassRegistry::Instance().Get("operator_activation_mkldnn_fuse_pass"); int original_nodes_num = graph->Nodes().size(); @@ -200,4 +201,4 @@ TEST(ConvActivationFusePass, conv_hard_sigmoid_fuse_pass) { } // namespace framework } // namespace paddle -USE_PASS(conv_activation_mkldnn_fuse_pass); +USE_PASS(operator_activation_mkldnn_fuse_pass); diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc deleted file mode 100644 index a96ce5e297a87e..00000000000000 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h" - -#include "paddle/fluid/framework/ir/graph_pattern_detector.h" -#include "paddle/fluid/framework/op_version_registry.h" -#include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/string/pretty_log.h" - -namespace paddle { -namespace framework { -namespace ir { - -using string::PrettyLogDetail; - -void ElementwiseActivationOneDNNPass::ApplyImpl(Graph *graph) const { - std::vector act_types = {"relu", - "tanh", - "leaky_relu", - "swish", - "hard_swish", - "sqrt", - "abs", - "clip", - "gelu", - "relu6", - "sigmoid"}; - std::vector elt_types = { - "elementwise_add", "elementwise_sub", "elementwise_mul"}; - - for (const auto &elt_type : elt_types) - for (const auto &act_type : act_types) { - std::unordered_map attr_map; - - if (act_type == "swish") - attr_map.emplace("beta", "activation_alpha"); - else if (act_type == "relu6") - attr_map.emplace("threshold", "activation_alpha"); - else if (act_type == "clip") { - attr_map.emplace("min", "activation_alpha"); - attr_map.emplace("max", "activation_beta"); - } else { - attr_map.emplace("alpha", "activation_alpha"); - attr_map.emplace("beta", "activation_beta"); - } - FuseElementwiseAct(graph, elt_type, act_type, attr_map); - } -} - -void ElementwiseActivationOneDNNPass::FuseElementwiseAct( - Graph *graph, - const std::string &elt_type, - const std::string &act_type, - const std::unordered_map &attr_map) const { - PADDLE_ENFORCE_NOT_NULL( - graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); - FusePassBase::Init(elt_type + "_" + act_type + "_mkldnn_fuse_pass", graph); - - GraphPatternDetector gpd; - auto *elementwise_input = gpd.mutable_pattern() - ->NewNode(elt_type + "_act/elementwise_input") - ->AsInput() - ->assert_is_op_input(elt_type, "X"); - patterns::ElementwiseActivation elementwise_act_pattern(gpd.mutable_pattern(), - elt_type + "_act"); - elementwise_act_pattern(elementwise_input, elt_type, act_type); - - int found_elementwise_activation_count = 0; - auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, - Graph *g) { - VLOG(4) << "Fuse " << elt_type << " with activation op."; - // Elementwise output - GET_IR_NODE_FROM_SUBGRAPH( - elementwise_out, elementwise_out, elementwise_act_pattern); - // ACT output - GET_IR_NODE_FROM_SUBGRAPH( - activation_out, activation_out, elementwise_act_pattern); - // ops - GET_IR_NODE_FROM_SUBGRAPH( - elementwise, elementwise, elementwise_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation, activation, elementwise_act_pattern); - - auto *elementwise_op = elementwise->Op(); - - if (elementwise_op->HasAttr("use_mkldnn")) { - const std::string wo_elt_type = - "The " + elt_type; // Workaround for PP error message checking. - PADDLE_ENFORCE_EQ( - BOOST_GET_CONST(bool, elementwise_op->GetAttr("use_mkldnn")), - true, - platform::errors::PreconditionNotMet( - wo_elt_type + "+Act fusion may happen only when oneDNN library " - "is used.")); - } - - auto *activation_op = activation->Op(); - for (const auto &attr : attr_map) { - if (activation_op->HasAttr(attr.first)) { - elementwise_op->SetAttr(attr.second, - activation_op->GetAttr(attr.first)); - } - } - - if (act_type == "gelu" && activation_op->HasAttr("approximate") && - BOOST_GET_CONST(bool, activation_op->GetAttr("approximate"))) - elementwise_op->SetAttr("activation_type", std::string("gelu_tanh")); - else - elementwise_op->SetAttr("activation_type", act_type); - - elementwise_op->SetOutput("Out", {activation_out->Name()}); - - IR_OP_VAR_LINK(elementwise, activation_out); - GraphSafeRemoveNodes(g, {activation, elementwise_out}); - found_elementwise_activation_count++; - }; - - gpd(graph, handler); - AddStatis(found_elementwise_activation_count); - PrettyLogDetail("--- fused %d %s with %s activation", - found_elementwise_activation_count, - elt_type, - act_type); -} - -} // namespace ir -} // namespace framework -} // namespace paddle - -REGISTER_PASS(elt_act_mkldnn_fuse_pass, - paddle::framework::ir::ElementwiseActivationOneDNNPass); -REGISTER_PASS_CAPABILITY(elt_act_mkldnn_fuse_pass) - .AddCombination( - paddle::framework::compatible::OpVersionComparatorCombination() - .LE("elementwise_add", 1) - .LE("elementwise_sub", 1) - .LE("elementwise_mul", 1) - .LE("relu", 0) - .LE("tanh", 0) - .LE("leaky_relu", 1) - .LE("swish", 0) - .LE("hard_swish", 0) - .LE("sqrt", 0) - .LE("abs", 0) - .LE("clip", 1) - .LE("gelu", 0) - .LE("relu6", 0) - .LE("sigmoid", 0)); diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h deleted file mode 100644 index 8df479e3ddf06d..00000000000000 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include "paddle/fluid/framework/ir/fuse_pass_base.h" -#include "paddle/fluid/framework/ir/graph.h" - -namespace paddle { -namespace framework { -namespace ir { - -/* - * \brief Fuse the Elementwise and activation operators into single - * OneDNN's Elementwise with post-op. - */ -class ElementwiseActivationOneDNNPass : public FusePassBase { - public: - virtual ~ElementwiseActivationOneDNNPass() {} - - protected: - void ApplyImpl(Graph *graph) const override; - - void FuseElementwiseAct( - Graph *graph, - const std::string &elt_types, - const std::string &act_types, - const std::unordered_map &attr_map) const; -}; - -} // namespace ir -} // namespace framework -} // namespace paddle diff --git a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc new file mode 100644 index 00000000000000..da2e8f64f20a1a --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc @@ -0,0 +1,174 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h" + +#include "paddle/fluid/framework/ir/graph_pattern_detector.h" +#include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/string/pretty_log.h" + +namespace paddle { +namespace framework { +namespace ir { + +using string::PrettyLogDetail; + +void OperatorActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { + std::vector act_types = {"relu", + "mish", + "swish", + "sqrt", + "hard_swish", + "sigmoid", + "abs", + "gelu", + "relu6", + "clip", + "tanh", + "hard_sigmoid", + "leaky_relu"}; + + std::vector op_types = {"conv2d", + "elementwise_add", + "elementwise_sub", + "elementwise_mul", + "matmul", + "softplus"}; + + for (const auto& op_type : op_types) + for (auto& act_type : act_types) { + std::unordered_map attrs_map; + + if (act_type == "swish") + attrs_map.emplace("beta", "fuse_alpha"); + else if (act_type == "relu6") + attrs_map.emplace("threshold", "fuse_alpha"); + else if (act_type == "hard_sigmoid") { + attrs_map.emplace("slope", "fuse_alpha"); + attrs_map.emplace("offset", "fuse_beta"); + } else if (act_type == "clip") { + attrs_map.emplace("min", "fuse_alpha"); + attrs_map.emplace("max", "fuse_beta"); + } else { + attrs_map.emplace("alpha", "fuse_alpha"); + attrs_map.emplace("beta", "fuse_beta"); + } + FuseOperatorAct(graph, op_type, act_type, attrs_map); + } +} + +void OperatorActivationMkldnnFusePass::FuseOperatorAct( + Graph* graph, + const std::string& op_type, + std::string& act_type, + const std::unordered_map& attrs_map) const { + PADDLE_ENFORCE_NOT_NULL( + graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); + FusePassBase::Init(op_type + "_" + act_type + "_mkldnn_fuse_pass", graph); + + GraphPatternDetector gpd; + patterns::OperatorActivation op_act_pattern( + gpd.mutable_pattern(), "operator_activation_mkldnn_fuse"); + op_act_pattern(op_type, act_type); + + int found_operator_activation_count = 0; + auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, + Graph* g) { + GET_IR_NODE_FROM_SUBGRAPH(preceding_op, preceding_op, op_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH( + preceding_op_out, preceding_op_out, op_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(activation, activation, op_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, op_act_pattern); + + OpDesc* activated_op = preceding_op->Op(); + OpDesc* act_op = activation->Op(); + + if (activated_op->HasAttr("use_mkldnn")) { + PADDLE_ENFORCE_EQ( + BOOST_GET_CONST(bool, activated_op->GetAttr("use_mkldnn")), + true, + platform::errors::PreconditionNotMet( + "Activation fusion may happen only " + "when oneDNN library is used.")); + } + + for (const auto& attrs : attrs_map) { + if (act_op->HasAttr(attrs.first)) { + activated_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); + } + } + + if (act_type == "gelu" && activation->Op()->HasAttr("approximate")) { + act_type = BOOST_GET_CONST(bool, activation->Op()->GetAttr("approximate")) + ? "gelu_tanh" + : "gelu_erf"; + activated_op->SetAttr("fuse_alpha", 0.0f); + activated_op->SetAttr("fuse_beta", 0.0f); + } + + std::string output_name = + (op_type == "conv2d" || op_type == "conv2d_transpose") ? "Output" + : "Out"; + + activated_op->SetAttr("fuse_activation", act_type); + activated_op->SetOutput(output_name, {activation_out->Name()}); + + IR_NODE_LINK_TO(preceding_op, activation_out); + GraphSafeRemoveNodes(graph, {activation, preceding_op_out}); + found_operator_activation_count++; + }; + + gpd(graph, handler); + AddStatis(found_operator_activation_count); + if ((!Has("disable_logs") || !Get("disable_logs")) && + found_operator_activation_count > 0) { + PrettyLogDetail("--- fused %d %s with %s activation", + found_operator_activation_count, + op_type, + act_type); + } +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +REGISTER_PASS(operator_activation_mkldnn_fuse_pass, + paddle::framework::ir::OperatorActivationMkldnnFusePass); + +REGISTER_PASS_CAPABILITY(operator_activation_mkldnn_fuse_pass) + .AddCombination( + paddle::framework::compatible::OpVersionComparatorCombination() + .LE("conv2d", 1) + .LE("conv2d_transpose", 2) + .LE("elementwise_add", 1) + .LE("elementwise_sub", 1) + .LE("elementwise_mul", 1) + .LE("matmul", 1) + .LE("softplus", 1) + + .EQ("abs", 0) + .LE("clip", 1) + .EQ("gelu", 0) + .EQ("hard_sigmoid", 0) + .LE("hard_swish", 0) + .LE("leaky_relu", 1) + .LE("mish", 1) + .EQ("relu", 0) + .EQ("relu6", 0) + .EQ("sigmoid", 0) + .EQ("sqrt", 0) + .EQ("swish", 0) + .EQ("tanh", 0)); \ No newline at end of file diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h similarity index 81% rename from paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h rename to paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h index e1e2898384609e..5323e058ecafd5 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h @@ -14,8 +14,6 @@ #pragma once -#include - #include "paddle/fluid/framework/ir/fuse_pass_base.h" #include "paddle/fluid/framework/ir/graph.h" @@ -23,21 +21,20 @@ namespace paddle { namespace framework { namespace ir { -class ConvActivationMkldnnFusePass : public FusePassBase { +class OperatorActivationMkldnnFusePass : public FusePassBase { public: - ConvActivationMkldnnFusePass(); - virtual ~ConvActivationMkldnnFusePass() {} + virtual ~OperatorActivationMkldnnFusePass() {} protected: void ApplyImpl(Graph *graph) const override; - void FuseConvAct( + void FuseOperatorAct( Graph *graph, - const std::string &conv_type, + const std::string &op_type, std::string &act_type, const std::unordered_map &attrs_map) const; }; } // namespace ir } // namespace framework -} // namespace paddle +} // namespace paddle \ No newline at end of file diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc deleted file mode 100644 index 3dd850d886c8e0..00000000000000 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h" - -#include "paddle/fluid/framework/ir/graph_pattern_detector.h" -#include "paddle/fluid/framework/op_version_registry.h" -#include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/string/pretty_log.h" - -namespace paddle { -namespace framework { -namespace ir { - -using string::PrettyLogDetail; - -void SoftplusActivationOneDNNPass::ApplyImpl(Graph *graph) const { - std::vector act_types = {"relu", - "tanh", - "leaky_relu", - "swish", - "hardswish", - "sqrt", - "abs", - "clip", - "gelu", - "relu6", - "sigmoid"}; - - for (const auto &act_type : act_types) { - std::unordered_map attr_map; - - if (act_type == "swish") - attr_map.emplace("beta", "fuse_activation_alpha"); - else if (act_type == "relu6") - attr_map.emplace("threshold", "fuse_activation_alpha"); - else if (act_type == "clip") { - attr_map.emplace("min", "fuse_activation_alpha"); - attr_map.emplace("max", "fuse_activation_beta"); - } else { - attr_map.emplace("alpha", "fuse_activation_alpha"); - attr_map.emplace("beta", "fuse_activation_beta"); - } - FuseSoftplusActivation(graph, act_type, attr_map); - } -} - -void SoftplusActivationOneDNNPass::FuseSoftplusActivation( - Graph *graph, - const std::string &fuse_activation_type, - const std::unordered_map &attr_map) const { - PADDLE_ENFORCE_NOT_NULL( - graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); - FusePassBase::Init("softplus_activation", graph); - - GraphPatternDetector gpd; - patterns::SoftplusActivation softplus_activation_pattern( - gpd.mutable_pattern(), "softplus_activation"); - softplus_activation_pattern(fuse_activation_type); - - int found_softplus_activation_count = 0; - auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, - Graph *g) { - VLOG(4) << "Fuse softplus with activation op."; - GET_IR_NODE_FROM_SUBGRAPH( - softplus_out, softplus_out, softplus_activation_pattern); - GET_IR_NODE_FROM_SUBGRAPH( - activation_out, activation_out, softplus_activation_pattern); - - GET_IR_NODE_FROM_SUBGRAPH(softplus, softplus, softplus_activation_pattern); - GET_IR_NODE_FROM_SUBGRAPH( - activation, activation, softplus_activation_pattern); - - auto *softplus_op = softplus->Op(); - - if (softplus_op->HasAttr("use_mkldnn")) { - PADDLE_ENFORCE_EQ( - BOOST_GET_CONST(bool, softplus_op->GetAttr("use_mkldnn")), - true, - platform::errors::PreconditionNotMet("The softplus + activation " - "fusion may happen only when " - "oneDNN library is used.")); - } - - auto *activation_op = activation->Op(); - for (const auto &attr : attr_map) { - if (activation_op->HasAttr(attr.first)) { - softplus_op->SetAttr(attr.second, activation_op->GetAttr(attr.first)); - } - } - - if (fuse_activation_type == "gelu" && - activation_op->HasAttr("approximate") && - BOOST_GET_CONST(bool, activation_op->GetAttr("approximate"))) - softplus_op->SetAttr("fuse_activation_type", std::string("gelu_tanh")); - else - softplus_op->SetAttr("fuse_activation_type", fuse_activation_type); - - softplus_op->SetAttr("use_mkldnn", true); - - softplus_op->SetOutput("Out", {activation_out->Name()}); - - IR_OP_VAR_LINK(softplus, activation_out); - GraphSafeRemoveNodes(g, {activation, softplus_out}); - found_softplus_activation_count++; - }; - - gpd(graph, handler); - AddStatis(found_softplus_activation_count); - if (!Has("disable_logs") || !Get("disable_logs")) - PrettyLogDetail("--- fused %d softplus with %s activation", - found_softplus_activation_count, - fuse_activation_type); -} -} // namespace ir -} // namespace framework -} // namespace paddle - -REGISTER_PASS(softplus_activation_mkldnn_fuse_pass, - paddle::framework::ir::SoftplusActivationOneDNNPass); -REGISTER_PASS_CAPABILITY(softplus_activation_mkldnn_fuse_pass) - .AddCombination( - paddle::framework::compatible::OpVersionComparatorCombination() - .LE("softplus", 1) - .EQ("relu", 0) - .EQ("tanh", 0) - .LE("leaky_relu", 1) - .EQ("swish", 0) - .EQ("hard_swish", 0) - .EQ("sqrt", 0) - .EQ("abs", 0) - .LE("relu6", 1) - .LE("clip", 1) - .EQ("gelu", 0)); diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h deleted file mode 100644 index c49502c674355a..00000000000000 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include "paddle/fluid/framework/ir/fuse_pass_base.h" -#include "paddle/fluid/framework/ir/graph.h" - -namespace paddle { -namespace framework { -namespace ir { - -/* - * \brief Fuse the softplus and another activation operators into - * softplus with another activation as post-op. - */ -class SoftplusActivationOneDNNPass : public FusePassBase { - public: - virtual ~SoftplusActivationOneDNNPass() {} - - protected: - void ApplyImpl(ir::Graph *graph) const override; - - void FuseSoftplusActivation( - ir::Graph *graph, - const std::string &fuse_activation_type, - const std::unordered_map &attr_map) const; -}; - -} // namespace ir -} // namespace framework -} // namespace paddle diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc index 2fbb46e32d1e97..64132cdc7e23c7 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc @@ -50,9 +50,9 @@ void MainTest(const std::string& activation_type) { const auto* op = node->Op(); ASSERT_TRUE(op->HasAttr("use_mkldnn")); EXPECT_TRUE(BOOST_GET_CONST(bool, op->GetAttr("use_mkldnn"))); - ASSERT_TRUE(op->HasAttr("fuse_activation_type")); + ASSERT_TRUE(op->HasAttr("fuse_activation")); auto activation_type = - BOOST_GET_CONST(std::string, op->GetAttr("fuse_activation_type")); + BOOST_GET_CONST(std::string, op->GetAttr("fuse_activation")); EXPECT_EQ(activation_type.compare(activation_type), 0); } } @@ -88,4 +88,4 @@ TEST(FuseSoftplusActivationOneDNNPass, FuseSoftplusWithSigmoid) { } // namespace framework } // namespace paddle -USE_PASS(softplus_activation_mkldnn_fuse_pass); +USE_PASS(operator_activation_mkldnn_fuse_pass); diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc index 954a6898781c18..9f97b20e972005 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.cc +++ b/paddle/fluid/inference/api/paddle_pass_builder.cc @@ -270,7 +270,7 @@ void CpuPassStrategy::EnableMKLDNN() { // "conv3d_bias_mkldnn_fuse_pass", // "conv_elementwise_add_mkldnn_fuse_pass", "conv_concat_relu_mkldnn_fuse_pass", - "conv_activation_mkldnn_fuse_pass", // + "operator_activation_mkldnn_fuse_pass", // "scale_matmul_fuse_pass", // "reshape_transpose_matmul_mkldnn_fuse_pass", // "reshape_transpose_matmul_v2_mkldnn_fuse_pass", // @@ -279,11 +279,9 @@ void CpuPassStrategy::EnableMKLDNN() { // Disabled due to topology-dependent speed-up // "fc_mkldnn_pass", // "fc_act_mkldnn_fuse_pass", - "fc_elementwise_add_mkldnn_fuse_pass", // - "batch_norm_act_fuse_pass", // - "softplus_activation_mkldnn_fuse_pass", // - "shuffle_channel_mkldnn_detect_pass", // - "elt_act_mkldnn_fuse_pass", // + "fc_elementwise_add_mkldnn_fuse_pass", // + "batch_norm_act_fuse_pass", // + "shuffle_channel_mkldnn_detect_pass", // // TODO(intel): Please fix the bug on windows. // https://github.com/PaddlePaddle/Paddle/issues/29710 // "mkldnn_inplace_pass", // This pass should be activated after @@ -363,15 +361,13 @@ void CpuPassStrategy::EnableMkldnnInt8() { passes_.push_back("conv_transpose_bias_mkldnn_fuse_pass"); passes_.push_back("conv_elementwise_add_mkldnn_fuse_pass"); passes_.push_back("conv_concat_relu_mkldnn_fuse_pass"); - passes_.push_back("conv_activation_mkldnn_fuse_pass"); + passes_.push_back("operator_activation_mkldnn_fuse_pass"); passes_.push_back("fc_fuse_pass"); passes_.push_back("repeated_fc_relu_fuse_pass"); passes_.push_back("fc_mkldnn_pass"); passes_.push_back("fc_act_mkldnn_fuse_pass"); passes_.push_back("matmul_transpose_reshape_fuse_pass"); passes_.push_back("matmul_v2_transpose_reshape_fuse_pass"); - passes_.push_back("batch_norm_act_fuse_pass"); - passes_.push_back("softplus_activation_mkldnn_fuse_pass"); passes_.push_back("compute_propagate_scales_mkldnn_pass"); passes_.push_back("scale_matmul_fuse_pass"); passes_.push_back("reshape_transpose_matmul_mkldnn_fuse_pass"); diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 0da38de5ead55d..e3eafa87a1a73c 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -492,26 +492,6 @@ class SoftplusOpMaker : public framework::OpProtoAndCheckerMaker { "(bool, default false) Only used in cudnn kernel, need install cudnn.") .SetDefault(false) .AsExtra(); - AddAttr( - "fuse_activation_type", - "Fused activation type used in softplus OneDNN kernel.") - .SetDefault("") - .AsExtra(); - AddAttr( - "fuse_activation_alpha", - "Fused activation alpha parameter type used in softplus OneDNN kernel.") - .SetDefault(0.0f) - .AsExtra(); - AddAttr( - "fuse_activation_beta", - "Fused activation beta parameter type used in softplus OneDNN kernel.") - .SetDefault(0.0f) - .AsExtra(); - AddAttr( - "fuse_activation_scale", - "Fused activation scale parameter type used in softplus OneDNN kernel.") - .SetDefault(1.0f) - .AsExtra(); AddComment(R"DOC( :strong:`Softplus Activation Operator` diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h index 61552e492dfa19..0639b7fbf145b7 100644 --- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h +++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h @@ -50,21 +50,28 @@ class EltwiseMKLDNNKernel : public framework::OpKernel { private: dnnl::post_ops get_post_ops(const framework::ExecutionContext& ctx) const { dnnl::post_ops post_operations; - if (ctx.HasAttr("activation_type")) { - const float scale = ctx.HasAttr("activation_scale") - ? ctx.Attr("activation_scale") - : 1.0f; - const float alpha = ctx.HasAttr("activation_alpha") - ? ctx.Attr("activation_alpha") - : 0.0f; - const float beta = ctx.HasAttr("activation_beta") - ? ctx.Attr("activation_beta") - : 0.0f; - - const auto activation_algorithm = platform::AcquireActivationAlgorithm( - ctx.Attr("activation_type")); - - post_operations.append_eltwise(scale, activation_algorithm, alpha, beta); + if (ctx.HasAttr("fuse_activation")) { + const auto fuse_activation = ctx.Attr("fuse_activation"); + const auto fuse_alpha = + ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; + const auto fuse_beta = + ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; + const auto activation_scale = ctx.HasAttr("activation_scale") + ? ctx.Attr("activation_scale") + : 1.0f; + if (fuse_activation == "hard_sigmoid") { + post_operations.append_eltwise(activation_scale, + dnnl::algorithm::eltwise_linear, + fuse_alpha, + fuse_beta); + post_operations.append_eltwise( + activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); + } else if (fuse_activation != "") { + const auto activation_algorithm = + platform::AcquireActivationAlgorithm(fuse_activation); + post_operations.append_eltwise( + activation_scale, activation_algorithm, fuse_alpha, fuse_beta); + } } return post_operations; } diff --git a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h index d2aa1cfc6bbf7b..c947d63c64203a 100644 --- a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h +++ b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h @@ -46,7 +46,28 @@ class SoftplusMKLDNNHandler 1.0f, dnnl::algorithm::eltwise_linear, 1.0f / beta, 0.0f); } - AppendFusedActivationIfExists(ctx, &post_ops); + if (ctx.HasAttr("fuse_activation")) { + const auto fuse_activation = ctx.Attr("fuse_activation"); + const auto fuse_alpha = + ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; + const auto fuse_beta = + ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; + const auto activation_scale = 1.0f; + + if (fuse_activation == "hard_sigmoid") { + post_ops.append_eltwise(activation_scale, + dnnl::algorithm::eltwise_linear, + fuse_alpha, + fuse_beta); + post_ops.append_eltwise( + activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); + } else if (fuse_activation != "") { + const auto activation_algorithm = + platform::AcquireActivationAlgorithm(fuse_activation); + post_ops.append_eltwise( + activation_scale, activation_algorithm, fuse_alpha, fuse_beta); + } + } dnnl::primitive_attr attrs; attrs.set_post_ops(post_ops); @@ -62,42 +83,8 @@ class SoftplusMKLDNNHandler return this->AcquireMemoryFromPrimitive( this->fwd_pd_->src1_desc(), platform::to_void_cast(beta)); } - - private: - void AppendFusedActivationIfExists(const framework::ExecutionContext& ctx, - dnnl::post_ops* post_ops) { - const auto& fused_activation_type = - algo_map.find(ctx.Attr("fuse_activation_type")); - - if (fused_activation_type != algo_map.end()) { - auto scale_out = - ctx.Attr("fuse_activation_scale"); // for future int8 support - post_ops->append_eltwise(scale_out, - fused_activation_type->second, - ctx.Attr("fuse_activation_alpha"), - ctx.Attr("fuse_activation_beta")); - } - } - - static const std::unordered_map algo_map; }; -template -const std::unordered_map - SoftplusMKLDNNHandler::algo_map = { - {"relu", dnnl::algorithm::eltwise_relu}, - {"tanh", dnnl::algorithm::eltwise_tanh}, - {"leaky_relu", dnnl::algorithm::eltwise_relu}, - {"swish", dnnl::algorithm::eltwise_swish}, - {"hardswish", dnnl::algorithm::eltwise_hardswish}, - {"sqrt", dnnl::algorithm::eltwise_sqrt}, - {"abs", dnnl::algorithm::eltwise_abs}, - {"clip", dnnl::algorithm::eltwise_clip}, - {"gelu", dnnl::algorithm::eltwise_gelu_erf}, - {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh}, - {"relu6", dnnl::algorithm::eltwise_bounded_relu}, - {"sigmoid", dnnl::algorithm::eltwise_logistic}}; - template void custom_softplus_eltwise_forward(const framework::ExecutionContext& ctx) { const auto& dev_ctx = diff --git a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py index 622d54343f6a0e..f50c69b170dc3d 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py @@ -432,7 +432,7 @@ def _optimize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'conv_transpose_bias_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_elementwise_add_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_concat_relu_mkldnn_fuse_pass') - graph = self._apply_pass(graph, 'conv_activation_mkldnn_fuse_pass') + graph = self._apply_pass(graph, 'operator_activation_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'fc_fuse_pass', ['use_gpu', 'use_fc_padding'], [False, False]) graph = self._apply_pass(graph, 'repeated_fc_relu_fuse_pass') @@ -443,7 +443,6 @@ def _optimize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'matmul_transpose_reshape_fuse_pass') graph = self._apply_pass(graph, 'matmul_v2_transpose_reshape_fuse_pass') graph = self._apply_pass(graph, 'batch_norm_act_fuse_pass') - graph = self._apply_pass(graph, 'softplus_activation_mkldnn_fuse_pass') # the following pass should be the last one since it will work on all fused ops. graph = self._apply_pass(graph, 'runtime_context_cache_pass') return graph diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py index 2a3e349a18a7d2..8e1c93020cb63e 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py @@ -170,7 +170,7 @@ def sample_program_config(self, draw): # 11. Generate legal attr of act act_op = None - self.passes = ["conv_activation_mkldnn_fuse_pass"] + self.passes = ["operator_activation_mkldnn_fuse_pass"] if act_type == "relu6": threshold = draw(st.floats(min_value=1.0, max_value=10.0)) act_op = OpConfig("relu6", diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py index 0c4bfc8ee26043..d0dbde35eb89e6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py @@ -40,7 +40,7 @@ def setUp(self): } self.fetch_list = [conv_out] self.enable_mkldnn = True - self.pass_name = 'conv_activation_mkldnn_fuse_pass' + self.pass_name = 'operator_activation_mkldnn_fuse_pass' def set_params(self): self.conv_num_filters = 3 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py index dff23e96dd244d..7019b193228c7d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py @@ -103,7 +103,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + passes=["operator_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py index 1eb325f1d08799..630ddf5879e7a5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py @@ -104,7 +104,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + passes=["operator_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py index a0ed73964f3770..fd788083d79e8c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py @@ -106,7 +106,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + passes=["operator_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py index 6f5810cb802d9f..4594cd3e86d343 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py @@ -100,7 +100,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + passes=["operator_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py index 12f4249a4d66df..da691ccb4c6af0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py @@ -25,7 +25,7 @@ class ElementwiseActivationMkldnnFusePassTest(InferencePassTest): act_alpha = None act_beta = None - pass_name = 'elt_act_mkldnn_fuse_pass' + pass_name = 'operator_activation_mkldnn_fuse_pass' def setUp(self): self.set_params() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py index 78393ef59b658c..a4a420e7b6a740 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py @@ -76,7 +76,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["elt_act_mkldnn_fuse_pass"], + passes=["operator_activation_mkldnn_fuse_pass"], min_success_num=4) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 0c25a790138cd5..089c3c6cb034cd 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,137 +12,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -import unittest +from auto_scan_test import PassAutoScanTest +from program_config import TensorConfig, ProgramConfig, OpConfig import numpy as np -from inference_pass_test import InferencePassTest -import paddle -import paddle.fluid as fluid -from paddle.fluid.core import PassVersionChecker - - -class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): - fuse_activation_alpha = None - fuse_activation_beta = None - pass_name = 'softplus_activation_mkldnn_fuse_pass' - - def setUp(self): - self.set_params() - with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") - softplus_out = fluid.layers.softplus(data) - if self.fuse_activation_beta is not None: - activation_out = self.fuse_activation( - softplus_out, self.fuse_activation_alpha, - self.fuse_activation_beta) - elif self.fuse_activation_alpha is not None: - activation_out = self.fuse_activation( - softplus_out, self.fuse_activation_alpha) - else: - activation_out = self.fuse_activation(softplus_out) - - self.feeds = { - "data": np.random.random((1, 3, 100, 100)).astype("float32"), - } - self.fetch_list = [activation_out] - self.enable_mkldnn = True - - def set_params(self): - self.fuse_activation = fluid.layers.relu - - def test_check_output(self): - use_gpu = False - self.check_output_with_option(use_gpu) - - def test_pass_compatible(self): - self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name)) - - -class SoftplusActivationTanhOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.tanh - - -class SoftplusActivationLeakyReluOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.leaky_relu - self.fuse_activation_alpha = 0.3 - - -class SoftplusActivationSwishOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.swish - self.fuse_activation_alpha = 3 - - -class SoftplusActivationHardSwishOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.hard_swish - - -class SoftplusActivationSqrtOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.hard_swish - - -class SoftplusActivationAbsOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.abs - - -class SoftplusActivationClipOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.clip - self.fuse_activation_alpha = 1.1 - self.fuse_activation_beta = 5.2 - - -class SoftplusActivationGeluErfOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.gelu - - -class SoftplusActivationGeluTanhOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.gelu - self.fuse_activation_alpha = True # simulated "Approximate" attr - - -class SoftplusActivationRelu6OneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.relu6 - - -class SoftplusActivationSigmoidOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - - def set_params(self): - self.fuse_activation = fluid.layers.sigmoid - - -if __name__ == "__main__": - paddle.enable_static() +from functools import partial +import unittest +import hypothesis.strategies as st + + +class TestSoftplusActivationMkldnnFusePass(PassAutoScanTest): + + def sample_program_config(self, draw): + activation_type = draw( + st.sampled_from([ + 'relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', + 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', + 'hard_sigmoid', 'leaky_relu' + ])) + + def generate_input(): + shape = [16, 3, 100, 100] + return np.random.random(shape).astype(np.float32) + + softplus_op = OpConfig(type='softplus', + inputs={ + 'X': ['softplus_X'] + }, + outputs={'Out': ['softplus_output']}, + attrs={ + 'beta': 1, + 'threshold': 20, + 'use_mkldnn': True + }) + + if activation_type == "relu6": + activation_op = OpConfig(activation_type, + inputs={"X": ["softplus_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=draw( + st.floats(min_value=1.0, + max_value=10.0))) + elif activation_type == "leaky_relu": + activation_op = OpConfig(activation_type, + inputs={"X": ["softplus_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw( + st.floats(min_value=0.1, + max_value=1.0))) + elif activation_type == "swish": + activation_op = OpConfig(activation_type, + inputs={"X": ["softplus_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw( + st.floats(min_value=0.1, + max_value=1.0))) + elif activation_type == "clip": + activation_op = OpConfig( + activation_type, + inputs={"X": ["softplus_output"]}, + outputs={"Out": ["activation_output"]}, + min=draw(st.floats(min_value=0.1, max_value=0.49)), + max=draw(st.floats(min_value=0.5, max_value=1.0))) + else: + activation_op = OpConfig(activation_type, + inputs={"X": ["softplus_output"]}, + outputs={"Out": ["activation_output"]}) + + model_net = [softplus_op, activation_op] + + program_config = ProgramConfig( + ops=model_net, + weights={}, + inputs={ + 'softplus_X': TensorConfig(data_gen=partial(generate_input)) + }, + outputs=['activation_output']) + + return program_config + + def sample_predictor_configs(self, program_config): + config = self.create_inference_config(use_mkldnn=True) + yield config, ['softplus'], (1e-5, 1e-5) + + def test(self): + self.run_and_statis(quant=False, + max_examples=100, + passes=['operator_activation_mkldnn_fuse_pass']) + + +if __name__ == '__main__': unittest.main() From 691a77c980311c896c433fcf0bd0c4d792053163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 05:37:03 -0700 Subject: [PATCH 14/27] change order --- .../mkldnn/operator_activation_mkldnn_fuse_pass.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc index da2e8f64f20a1a..5167f54ea0bacc 100644 --- a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc @@ -26,6 +26,12 @@ namespace ir { using string::PrettyLogDetail; void OperatorActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { + std::vector op_types = {"conv2d", + "elementwise_add", + "elementwise_sub", + "elementwise_mul", + "matmul", + "softplus"}; std::vector act_types = {"relu", "mish", "swish", @@ -39,14 +45,6 @@ void OperatorActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { "tanh", "hard_sigmoid", "leaky_relu"}; - - std::vector op_types = {"conv2d", - "elementwise_add", - "elementwise_sub", - "elementwise_mul", - "matmul", - "softplus"}; - for (const auto& op_type : op_types) for (auto& act_type : act_types) { std::unordered_map attrs_map; From 9a843665ad706de4a25460ac88e64709ac1ba32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 16:35:16 -0700 Subject: [PATCH 15/27] Unify activation GPD --- .../framework/ir/graph_pattern_detector.cc | 127 ++---------------- .../framework/ir/graph_pattern_detector.h | 98 +------------- .../conv_activation_mkldnn_fuse_pass.cc | 17 +-- .../ir/mkldnn/elt_act_mkldnn_fuse_pass.cc | 22 ++- .../ir/mkldnn/fc_act_mkldnn_fuse_pass.cc | 16 +-- .../matmul_activation_mkldnn_fuse_pass.cc | 8 +- .../softplus_activation_mkldnn_fuse_pass.cc | 13 +- ...test_mkldnn_matmul_activation_fuse_pass.py | 2 +- 8 files changed, 53 insertions(+), 250 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index bd6a8c401262cb..f84ae402dc9924 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -925,83 +925,22 @@ PDNode *patterns::ConvBN::operator()(paddle::framework::ir::PDNode *conv_input, return bn_out_var; } -PDNode *patterns::ConvActivation::operator()( - paddle::framework::ir::PDNode *conv_input, - std::string conv_type, - std::string activation_type) { - // Create Operators - conv_input->assert_is_op_input(conv_type, "Input"); - auto *conv_op = pattern->NewNode(conv_repr())->assert_is_op(conv_type); - auto *activation_op = - pattern->NewNode(activation_repr())->assert_is_op(activation_type); - // Create variables - // Filter - auto *conv_weight_var = pattern->NewNode(conv_weight_repr()) - ->AsInput() - ->assert_is_persistable_var() - ->assert_is_op_input(conv_type, "Filter"); - // intermediate variable, will be removed in the IR after fuse. - auto *conv_out_var = pattern->NewNode(conv_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op(conv_type) - ->assert_is_op_input(activation_type); - // output - auto *activation_out_var = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - - conv_op->LinksFrom({conv_input, conv_weight_var}).LinksTo({conv_out_var}); - activation_op->LinksFrom({conv_out_var}).LinksTo({activation_out_var}); - return activation_out_var; -} - -PDNode *patterns::ElementwiseActivation::operator()( - paddle::framework::ir::PDNode *elementwise_a, - const std::string &elementwise_type, - const std::string &activation_type) { - // Create Operators - elementwise_a->assert_is_op_input(elementwise_type, "X"); - auto *elementwise_op = - pattern->NewNode(elementwise_repr())->assert_is_op(elementwise_type); +PDNode *patterns::OperatorActivation::operator()( + const std::string &operator_type, const std::string &activation_type) { + auto *preceding_op = + pattern->NewNode(preceding_op_repr())->assert_is_op(operator_type); + auto *preceding_op_out = pattern->NewNode(preceding_op_out_repr()) + ->AsIntermediate() + ->assert_is_only_output_of_op(operator_type) + ->assert_is_op_input(activation_type); auto *activation_op = pattern->NewNode(activation_repr())->assert_is_op(activation_type); - // Create variables - auto *elementwise_b = pattern->NewNode(elementwise_b_repr()) - ->AsInput() - ->assert_is_op_input(elementwise_type, "Y"); - // intermediate variable, will be removed in the IR after fuse. - auto *elementwise_out_var = - pattern->NewNode(elementwise_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op(elementwise_type) - ->assert_is_op_input(activation_type); - // output - auto *activation_out_var = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - - elementwise_op->LinksFrom({elementwise_a, elementwise_b}) - .LinksTo({elementwise_out_var}); - activation_op->LinksFrom({elementwise_out_var}).LinksTo({activation_out_var}); - return activation_out_var; -} - -PDNode *patterns::MatmulActivation::operator()( - const std::string &matmul_type, const std::string &activation_type) { - auto *matmul_op = - pattern->NewNode(matmul_op_repr())->assert_is_op(matmul_type); - auto *matmul_out_var = pattern->NewNode(matmul_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op(matmul_type) - ->assert_is_op_input(activation_type); - auto *activation_op = - pattern->NewNode(activation_op_repr())->assert_is_op(activation_type); - auto *activation_out_var = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - matmul_op->LinksTo({matmul_out_var}); - activation_op->LinksFrom({matmul_out_var}).LinksTo({activation_out_var}); - return activation_out_var; + auto *activation_out = pattern->NewNode(activation_out_repr()) + ->AsOutput() + ->assert_is_op_output(activation_type); + preceding_op->LinksTo({preceding_op_out}); + activation_op->LinksFrom({preceding_op_out}).LinksTo({activation_out}); + return activation_out; } PDNode *patterns::SeqConvEltAddRelu::operator()( @@ -1133,44 +1072,6 @@ PDNode *patterns::FCMKLDNN::operator()(paddle::framework::ir::PDNode *x, return fc_out_var; } -PDNode *patterns::FCActOneDNN::operator()(const std::string &act_type) { - auto *fc = pattern->NewNode(fc_repr())->assert_is_op("fc"); - auto *fc_out = pattern->NewNode(fc_out_repr()) - ->assert_is_op_output("fc", "Out") - ->assert_is_op_input(act_type); - auto *act = - pattern->NewNode(act_repr())->assert_is_op(act_type)->AsIntermediate(); - auto *act_out = pattern->NewNode(act_out_repr()) - ->assert_is_op_output(act_type, "Out") - ->AsOutput(); - - fc->LinksTo({fc_out}); - act->LinksFrom({fc_out}).LinksTo({act_out}); - - return act_out; -} - -PDNode *patterns::SoftplusActivation::operator()(std::string activation_type) { - // Create Operators - auto *softplus_op = - pattern->NewNode(softplus_repr())->assert_is_op("softplus"); - auto *activation_op = - pattern->NewNode(activation_repr())->assert_is_op(activation_type); - // intermediate variable, will be removed in the IR after fuse. - auto *softplus_out = pattern->NewNode(softplus_out_repr()) - ->AsIntermediate() - ->assert_is_only_output_of_op("softplus") - ->assert_is_op_input(activation_type); - // output - auto *activation_out = pattern->NewNode(activation_out_repr()) - ->AsOutput() - ->assert_is_op_output(activation_type); - - softplus_op->LinksTo({softplus_out}); - activation_op->LinksFrom({softplus_out}).LinksTo({activation_out}); - return activation_out; -} - PDNode *patterns::Embedding::operator()(PDNode *x) { x->assert_is_op_input("lookup_table", "Ids"); auto *lookup_table_op = diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index 70bf44eff436a9..9d661a67afcbb4 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -516,63 +516,16 @@ struct ConvBN : public PatternBase { PATTERN_DECL_NODE(bn_saved_variance); }; -// Conv with Activation -// op: conv + activation -// named nodes: -// conv_input, conv_weight, -// conv_out, conv, -// activation_out, activation -struct ConvActivation : public PatternBase { - ConvActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "conv_activation") {} +struct OperatorActivation : public PatternBase { + OperatorActivation(PDPattern* pattern, const std::string& name_scope) + : PatternBase(pattern, name_scope, "operator_activation") {} - PDNode* operator()(PDNode* conv_input, - std::string conv_type = "conv2d", - std::string activation_type = "relu"); - - // declare operator node's name - PATTERN_DECL_NODE(conv); - PATTERN_DECL_NODE(activation); - // declare variable node's name - PATTERN_DECL_NODE(conv_weight); - PATTERN_DECL_NODE(conv_out); - PATTERN_DECL_NODE(activation_out); -}; - -// Elementwise with Activation -// op: elementwise + activation -// named nodes: -// elementwise_a, elementwise_b, -// elementwise_out, elementwise, -// activation_out, activation -struct ElementwiseActivation : public PatternBase { - ElementwiseActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "elementwise_add_activation") {} - - PDNode* operator()(PDNode* elementwise_a, - const std::string& elementwise_type, + PDNode* operator()(const std::string& operator_type, const std::string& activation_type); - // declare operator node's name - PATTERN_DECL_NODE(elementwise); + PATTERN_DECL_NODE(preceding_op); + PATTERN_DECL_NODE(preceding_op_out); PATTERN_DECL_NODE(activation); - // declare variable node's name - PATTERN_DECL_NODE(elementwise_b); - PATTERN_DECL_NODE(elementwise_out); - PATTERN_DECL_NODE(activation_out); -}; - -// Matmul with Activation -struct MatmulActivation : public PatternBase { - MatmulActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "matmul_activation") {} - - PDNode* operator()(const std::string& matmul_type, - const std::string& activation_type); - - PATTERN_DECL_NODE(matmul_op); - PATTERN_DECL_NODE(matmul_out); - PATTERN_DECL_NODE(activation_op); PATTERN_DECL_NODE(activation_out); }; @@ -645,45 +598,6 @@ struct FCMKLDNN : public PatternBase { PATTERN_DECL_NODE(output); }; -// -// \brief Pattern looking for fc and a directly following activation -// operator. -// -// \note Currently only gelu and tanh are supported as an activation -// function. -// Formula: act(fc(x)) -// Op: fc + act -struct FCActOneDNN : public PatternBase { - FCActOneDNN(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "fc_act_onednn") {} - - PDNode* operator()(const std::string& act_type); - - // declare operator node's name - PATTERN_DECL_NODE(fc); - PATTERN_DECL_NODE(act); - PATTERN_DECL_NODE(fc_out); - PATTERN_DECL_NODE(act_out); -}; - -// Fuse softplus with activation -// ops: softplus + activation -// nodes: -// softplus, softplus_out, -// activation, activation_out -struct SoftplusActivation : public PatternBase { - SoftplusActivation(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "softplus_activation") {} - - PDNode* operator()(std::string activation_type); - - // declare operator node's name - PATTERN_DECL_NODE(softplus); - PATTERN_DECL_NODE(activation); - PATTERN_DECL_NODE(softplus_out); - PATTERN_DECL_NODE(activation_out); -}; - // Embedding struct Embedding : public PatternBase { Embedding(PDPattern* pattern, const std::string& name_scope) diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc index 4eefc2987bcb44..8cf31239399539 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc @@ -72,13 +72,9 @@ void ConvActivationMkldnnFusePass::FuseConvAct( FusePassBase::Init(conv_type + "_" + act_type + "_mkldnn_fuse_pass", graph); GraphPatternDetector gpd; - auto* conv_input = gpd.mutable_pattern() - ->NewNode("conv_activation_mkldnn_fuse/conv_input") - ->AsInput() - ->assert_is_op_input(conv_type, "Input"); - patterns::ConvActivation conv_act_pattern(gpd.mutable_pattern(), - "conv_activation_mkldnn_fuse"); - conv_act_pattern(conv_input, conv_type, act_type); + patterns::OperatorActivation conv_act_pattern(gpd.mutable_pattern(), + "conv_activation_mkldnn_fuse"); + conv_act_pattern(conv_type, act_type); int found_conv_activation_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, @@ -90,11 +86,10 @@ void ConvActivationMkldnnFusePass::FuseConvAct( return; } - GET_IR_NODE_FROM_SUBGRAPH(conv_weight, conv_weight, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(conv_out, conv_out, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(conv, conv, conv_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, conv_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(conv, preceding_op, conv_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(conv_out, preceding_op_out, conv_act_pattern); GET_IR_NODE_FROM_SUBGRAPH(activation, activation, conv_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, conv_act_pattern); OpDesc* conv_op = conv->Op(); OpDesc* act_op = activation->Op(); diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc index a96ce5e297a87e..3867ff4ee75b85 100644 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc @@ -69,28 +69,22 @@ void ElementwiseActivationOneDNNPass::FuseElementwiseAct( FusePassBase::Init(elt_type + "_" + act_type + "_mkldnn_fuse_pass", graph); GraphPatternDetector gpd; - auto *elementwise_input = gpd.mutable_pattern() - ->NewNode(elt_type + "_act/elementwise_input") - ->AsInput() - ->assert_is_op_input(elt_type, "X"); - patterns::ElementwiseActivation elementwise_act_pattern(gpd.mutable_pattern(), - elt_type + "_act"); - elementwise_act_pattern(elementwise_input, elt_type, act_type); + patterns::OperatorActivation elementwise_act_pattern(gpd.mutable_pattern(), + elt_type + "_act"); + elementwise_act_pattern(elt_type, act_type); int found_elementwise_activation_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse " << elt_type << " with activation op."; - // Elementwise output - GET_IR_NODE_FROM_SUBGRAPH( - elementwise_out, elementwise_out, elementwise_act_pattern); - // ACT output + GET_IR_NODE_FROM_SUBGRAPH( - activation_out, activation_out, elementwise_act_pattern); - // ops + elementwise, preceding_op, elementwise_act_pattern); GET_IR_NODE_FROM_SUBGRAPH( - elementwise, elementwise, elementwise_act_pattern); + elementwise_out, preceding_op_out, elementwise_act_pattern); GET_IR_NODE_FROM_SUBGRAPH(activation, activation, elementwise_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH( + activation_out, activation_out, elementwise_act_pattern); auto *elementwise_op = elementwise->Op(); diff --git a/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc index 99243ec7d7047b..34a95b207ba937 100644 --- a/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc @@ -39,20 +39,18 @@ void FuseFCActOneDNNPass::FuseFCAct(Graph *graph, FusePassBase::Init("fc_act", graph); GraphPatternDetector gpd; - patterns::FCActOneDNN fc_act_pattern(gpd.mutable_pattern(), "fc_act"); - fc_act_pattern(act_type); + patterns::OperatorActivation fc_act_pattern(gpd.mutable_pattern(), "fc_act"); + fc_act_pattern("fc", act_type); int found_fc_act_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse fc with activation op."; - // FC output - GET_IR_NODE_FROM_SUBGRAPH(fc_out, fc_out, fc_act_pattern); - // ACT output - GET_IR_NODE_FROM_SUBGRAPH(act_out, act_out, fc_act_pattern); - // ops - GET_IR_NODE_FROM_SUBGRAPH(fc, fc, fc_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(act, act, fc_act_pattern); + + GET_IR_NODE_FROM_SUBGRAPH(fc, preceding_op, fc_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(fc_out, preceding_op_out, fc_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(act, activation, fc_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(act_out, activation_out, fc_act_pattern); auto *fc_op = fc->Op(); auto *act_op = act->Op(); diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index 901e305ff90d04..80a68708d51592 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -72,7 +72,7 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( FusePassBase::Init(matmul_type + "_" + act_type + "_mkldnn_fuse_pass", graph); GraphPatternDetector gpd; - patterns::MatmulActivation matmul_act_pattern( + patterns::OperatorActivation matmul_act_pattern( gpd.mutable_pattern(), "matmul_activation_mkldnn_fuse"); matmul_act_pattern(matmul_type, act_type); @@ -86,9 +86,9 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( return; } - GET_IR_NODE_FROM_SUBGRAPH(matmul, matmul_op, matmul_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(matmul_out, matmul_out, matmul_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation, activation_op, matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(matmul, preceding_op, matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(matmul_out, preceding_op_out, matmul_act_pattern); + GET_IR_NODE_FROM_SUBGRAPH(activation, activation, matmul_act_pattern); GET_IR_NODE_FROM_SUBGRAPH( activation_out, activation_out, matmul_act_pattern); diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc index 3dd850d886c8e0..b816aa6b7af131 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc @@ -65,22 +65,23 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( FusePassBase::Init("softplus_activation", graph); GraphPatternDetector gpd; - patterns::SoftplusActivation softplus_activation_pattern( + patterns::OperatorActivation softplus_activation_pattern( gpd.mutable_pattern(), "softplus_activation"); - softplus_activation_pattern(fuse_activation_type); + softplus_activation_pattern("softplus", fuse_activation_type); int found_softplus_activation_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse softplus with activation op."; + GET_IR_NODE_FROM_SUBGRAPH( - softplus_out, softplus_out, softplus_activation_pattern); + softplus, preceding_op, softplus_activation_pattern); GET_IR_NODE_FROM_SUBGRAPH( - activation_out, activation_out, softplus_activation_pattern); - - GET_IR_NODE_FROM_SUBGRAPH(softplus, softplus, softplus_activation_pattern); + softplus_out, preceding_op_out, softplus_activation_pattern); GET_IR_NODE_FROM_SUBGRAPH( activation, activation, softplus_activation_pattern); + GET_IR_NODE_FROM_SUBGRAPH( + activation_out, activation_out, softplus_activation_pattern); auto *softplus_op = softplus->Op(); diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index 259112d43cb3b5..06488e5e769040 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -119,7 +119,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - max_examples=300, + max_examples=100, passes=['matmul_activation_mkldnn_fuse_pass']) From e6ac2140278e77afbfa248f41dae4c5ebd5fe1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 17:02:22 -0700 Subject: [PATCH 16/27] Revert changes with op_act --- paddle/fluid/framework/ir/CMakeLists.txt | 6 +- ...s.h => conv_activation_mkldnn_fuse_pass.h} | 13 +- ...conv_activation_mkldnn_fuse_pass_tester.cc | 7 +- .../operator_activation_mkldnn_fuse_pass.cc | 172 ------------- .../softplus_activation_mkldnn_fuse_pass.cc | 1 - .../softplus_activation_mkldnn_fuse_pass.h | 45 ++++ ...plus_activation_mkldnn_fuse_pass_tester.cc | 2 +- .../inference/api/paddle_pass_builder.cc | 15 +- .../mkldnn/elementwise_mkldnn_op.h | 37 ++- .../quantization/quant2_int8_mkldnn_pass.py | 3 +- .../test_conv_act_mkldnn_fuse_pass.py | 2 +- .../test_mkldnn_conv_activation_fuse_pass.py | 2 +- .../test_mkldnn_conv_gelu_fuse_pass.py | 5 +- ...test_mkldnn_conv_hard_sigmoid_fuse_pass.py | 2 +- .../test_mkldnn_conv_hard_swish_fuse_pass.py | 2 +- .../test_mkldnn_conv_mish_fuse_pass.py | 2 +- .../test_mkldnn_elt_act_fuse_pass.py | 2 +- .../test_mkldnn_elt_act_fuse_pass_new.py | 2 +- ...st_mkldnn_softplus_activation_fuse_pass.py | 231 ++++++++++-------- 19 files changed, 231 insertions(+), 320 deletions(-) rename paddle/fluid/framework/ir/mkldnn/{operator_activation_mkldnn_fuse_pass.h => conv_activation_mkldnn_fuse_pass.h} (81%) delete mode 100644 paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc create mode 100644 paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index b9d6ba58d6bc37..d19b163817e41e 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -197,7 +197,7 @@ if(WITH_MKLDNN) pass_library(depthwise_conv_mkldnn_pass base DIR mkldnn) pass_library(conv_affine_channel_mkldnn_fuse_pass inference DIR mkldnn) pass_library(conv_bias_mkldnn_fuse_pass inference DIR mkldnn) - pass_library(operator_activation_mkldnn_fuse_pass inference DIR mkldnn) + pass_library(conv_activation_mkldnn_fuse_pass inference DIR mkldnn) pass_library(conv_concat_relu_mkldnn_fuse_pass inference DIR mkldnn) pass_library(conv_elementwise_add_mkldnn_fuse_pass inference DIR mkldnn) pass_library(int8_scale_calculation_mkldnn_pass inference DIR mkldnn) @@ -208,8 +208,10 @@ if(WITH_MKLDNN) pass_library(cpu_bfloat16_pass inference DIR mkldnn) pass_library(fc_mkldnn_pass inference DIR mkldnn) pass_library(interpolate_mkldnn_pass inference DIR mkldnn) + pass_library(softplus_activation_mkldnn_fuse_pass inference DIR mkldnn) pass_library(shuffle_channel_mkldnn_detect_pass inference DIR mkldnn) pass_library(fc_act_mkldnn_fuse_pass inference DIR mkldnn) + pass_library(elt_act_mkldnn_fuse_pass inference DIR mkldnn) pass_library(cpu_quantize_placement_pass base DIR mkldnn) pass_library(cpu_quantize_pass inference DIR mkldnn) pass_library(cpu_quantize_squash_pass inference DIR mkldnn) @@ -402,7 +404,7 @@ if(WITH_MKLDNN) cc_test( test_conv_activation_mkldnn_fuse_pass SRCS mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc - DEPS operator_activation_mkldnn_fuse_pass) + DEPS conv_activation_mkldnn_fuse_pass) cc_test( test_conv_concat_relu_mkldnn_fuse_pass SRCS mkldnn/conv_concat_relu_mkldnn_fuse_pass_tester.cc diff --git a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h similarity index 81% rename from paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h rename to paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h index 5323e058ecafd5..e1e2898384609e 100644 --- a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h @@ -14,6 +14,8 @@ #pragma once +#include + #include "paddle/fluid/framework/ir/fuse_pass_base.h" #include "paddle/fluid/framework/ir/graph.h" @@ -21,20 +23,21 @@ namespace paddle { namespace framework { namespace ir { -class OperatorActivationMkldnnFusePass : public FusePassBase { +class ConvActivationMkldnnFusePass : public FusePassBase { public: - virtual ~OperatorActivationMkldnnFusePass() {} + ConvActivationMkldnnFusePass(); + virtual ~ConvActivationMkldnnFusePass() {} protected: void ApplyImpl(Graph *graph) const override; - void FuseOperatorAct( + void FuseConvAct( Graph *graph, - const std::string &op_type, + const std::string &conv_type, std::string &act_type, const std::unordered_map &attrs_map) const; }; } // namespace ir } // namespace framework -} // namespace paddle \ No newline at end of file +} // namespace paddle diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc index 05e6d8a058f24a..d75874c6de7120 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass_tester.cc @@ -16,7 +16,7 @@ #include -#include "paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h" +#include "paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h" #include "paddle/fluid/framework/op_proto_maker.h" namespace paddle { @@ -147,8 +147,7 @@ void MainTest(std::string activation) { std::unique_ptr graph(new ir::Graph(prog)); - auto pass = - PassRegistry::Instance().Get("operator_activation_mkldnn_fuse_pass"); + auto pass = PassRegistry::Instance().Get("conv_activation_mkldnn_fuse_pass"); int original_nodes_num = graph->Nodes().size(); @@ -201,4 +200,4 @@ TEST(ConvActivationFusePass, conv_hard_sigmoid_fuse_pass) { } // namespace framework } // namespace paddle -USE_PASS(operator_activation_mkldnn_fuse_pass); +USE_PASS(conv_activation_mkldnn_fuse_pass); diff --git a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc deleted file mode 100644 index 5167f54ea0bacc..00000000000000 --- a/paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.cc +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/framework/ir/mkldnn/operator_activation_mkldnn_fuse_pass.h" - -#include "paddle/fluid/framework/ir/graph_pattern_detector.h" -#include "paddle/fluid/framework/op_version_registry.h" -#include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/string/pretty_log.h" - -namespace paddle { -namespace framework { -namespace ir { - -using string::PrettyLogDetail; - -void OperatorActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - std::vector op_types = {"conv2d", - "elementwise_add", - "elementwise_sub", - "elementwise_mul", - "matmul", - "softplus"}; - std::vector act_types = {"relu", - "mish", - "swish", - "sqrt", - "hard_swish", - "sigmoid", - "abs", - "gelu", - "relu6", - "clip", - "tanh", - "hard_sigmoid", - "leaky_relu"}; - for (const auto& op_type : op_types) - for (auto& act_type : act_types) { - std::unordered_map attrs_map; - - if (act_type == "swish") - attrs_map.emplace("beta", "fuse_alpha"); - else if (act_type == "relu6") - attrs_map.emplace("threshold", "fuse_alpha"); - else if (act_type == "hard_sigmoid") { - attrs_map.emplace("slope", "fuse_alpha"); - attrs_map.emplace("offset", "fuse_beta"); - } else if (act_type == "clip") { - attrs_map.emplace("min", "fuse_alpha"); - attrs_map.emplace("max", "fuse_beta"); - } else { - attrs_map.emplace("alpha", "fuse_alpha"); - attrs_map.emplace("beta", "fuse_beta"); - } - FuseOperatorAct(graph, op_type, act_type, attrs_map); - } -} - -void OperatorActivationMkldnnFusePass::FuseOperatorAct( - Graph* graph, - const std::string& op_type, - std::string& act_type, - const std::unordered_map& attrs_map) const { - PADDLE_ENFORCE_NOT_NULL( - graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); - FusePassBase::Init(op_type + "_" + act_type + "_mkldnn_fuse_pass", graph); - - GraphPatternDetector gpd; - patterns::OperatorActivation op_act_pattern( - gpd.mutable_pattern(), "operator_activation_mkldnn_fuse"); - op_act_pattern(op_type, act_type); - - int found_operator_activation_count = 0; - auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, - Graph* g) { - GET_IR_NODE_FROM_SUBGRAPH(preceding_op, preceding_op, op_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH( - preceding_op_out, preceding_op_out, op_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation, activation, op_act_pattern); - GET_IR_NODE_FROM_SUBGRAPH(activation_out, activation_out, op_act_pattern); - - OpDesc* activated_op = preceding_op->Op(); - OpDesc* act_op = activation->Op(); - - if (activated_op->HasAttr("use_mkldnn")) { - PADDLE_ENFORCE_EQ( - BOOST_GET_CONST(bool, activated_op->GetAttr("use_mkldnn")), - true, - platform::errors::PreconditionNotMet( - "Activation fusion may happen only " - "when oneDNN library is used.")); - } - - for (const auto& attrs : attrs_map) { - if (act_op->HasAttr(attrs.first)) { - activated_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); - } - } - - if (act_type == "gelu" && activation->Op()->HasAttr("approximate")) { - act_type = BOOST_GET_CONST(bool, activation->Op()->GetAttr("approximate")) - ? "gelu_tanh" - : "gelu_erf"; - activated_op->SetAttr("fuse_alpha", 0.0f); - activated_op->SetAttr("fuse_beta", 0.0f); - } - - std::string output_name = - (op_type == "conv2d" || op_type == "conv2d_transpose") ? "Output" - : "Out"; - - activated_op->SetAttr("fuse_activation", act_type); - activated_op->SetOutput(output_name, {activation_out->Name()}); - - IR_NODE_LINK_TO(preceding_op, activation_out); - GraphSafeRemoveNodes(graph, {activation, preceding_op_out}); - found_operator_activation_count++; - }; - - gpd(graph, handler); - AddStatis(found_operator_activation_count); - if ((!Has("disable_logs") || !Get("disable_logs")) && - found_operator_activation_count > 0) { - PrettyLogDetail("--- fused %d %s with %s activation", - found_operator_activation_count, - op_type, - act_type); - } -} - -} // namespace ir -} // namespace framework -} // namespace paddle - -REGISTER_PASS(operator_activation_mkldnn_fuse_pass, - paddle::framework::ir::OperatorActivationMkldnnFusePass); - -REGISTER_PASS_CAPABILITY(operator_activation_mkldnn_fuse_pass) - .AddCombination( - paddle::framework::compatible::OpVersionComparatorCombination() - .LE("conv2d", 1) - .LE("conv2d_transpose", 2) - .LE("elementwise_add", 1) - .LE("elementwise_sub", 1) - .LE("elementwise_mul", 1) - .LE("matmul", 1) - .LE("softplus", 1) - - .EQ("abs", 0) - .LE("clip", 1) - .EQ("gelu", 0) - .EQ("hard_sigmoid", 0) - .LE("hard_swish", 0) - .LE("leaky_relu", 1) - .LE("mish", 1) - .EQ("relu", 0) - .EQ("relu6", 0) - .EQ("sigmoid", 0) - .EQ("sqrt", 0) - .EQ("swish", 0) - .EQ("tanh", 0)); \ No newline at end of file diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc index b816aa6b7af131..d6b482714679a9 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc @@ -73,7 +73,6 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse softplus with activation op."; - GET_IR_NODE_FROM_SUBGRAPH( softplus, preceding_op, softplus_activation_pattern); GET_IR_NODE_FROM_SUBGRAPH( diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h new file mode 100644 index 00000000000000..3d464a52deffbe --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h @@ -0,0 +1,45 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph.h" + +namespace paddle { +namespace framework { +namespace ir { + +/* + * \brief Fuse the softplus and another activation operators into + * softplus with another activation as post-op. + */ +class SoftplusActivationOneDNNPass : public FusePassBase { + public: + virtual ~SoftplusActivationOneDNNPass() {} + + protected: + void ApplyImpl(ir::Graph *graph) const override; + + void FuseSoftplusActivation( + ir::Graph *graph, + const std::string &fuse_activation, + const std::unordered_map &attr_map) const; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle \ No newline at end of file diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc index 64132cdc7e23c7..afe3d75fd2126f 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass_tester.cc @@ -88,4 +88,4 @@ TEST(FuseSoftplusActivationOneDNNPass, FuseSoftplusWithSigmoid) { } // namespace framework } // namespace paddle -USE_PASS(operator_activation_mkldnn_fuse_pass); +USE_PASS(softplus_activation_mkldnn_fuse_pass); diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc index 9f97b20e972005..ceaf88f8ca78ce 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.cc +++ b/paddle/fluid/inference/api/paddle_pass_builder.cc @@ -270,7 +270,7 @@ void CpuPassStrategy::EnableMKLDNN() { // "conv3d_bias_mkldnn_fuse_pass", // "conv_elementwise_add_mkldnn_fuse_pass", "conv_concat_relu_mkldnn_fuse_pass", - "operator_activation_mkldnn_fuse_pass", // + "conv_activation_mkldnn_fuse_pass", // "scale_matmul_fuse_pass", // "reshape_transpose_matmul_mkldnn_fuse_pass", // "reshape_transpose_matmul_v2_mkldnn_fuse_pass", // @@ -279,9 +279,12 @@ void CpuPassStrategy::EnableMKLDNN() { // Disabled due to topology-dependent speed-up // "fc_mkldnn_pass", // "fc_act_mkldnn_fuse_pass", - "fc_elementwise_add_mkldnn_fuse_pass", // - "batch_norm_act_fuse_pass", // - "shuffle_channel_mkldnn_detect_pass", // + "fc_elementwise_add_mkldnn_fuse_pass", // + "batch_norm_act_fuse_pass", // + "softplus_activation_mkldnn_fuse_pass", // + "shuffle_channel_mkldnn_detect_pass", // + "elt_act_mkldnn_fuse_pass", // + "matmul_activation_mkldnn_fuse_pass", // // TODO(intel): Please fix the bug on windows. // https://github.com/PaddlePaddle/Paddle/issues/29710 // "mkldnn_inplace_pass", // This pass should be activated after @@ -361,13 +364,15 @@ void CpuPassStrategy::EnableMkldnnInt8() { passes_.push_back("conv_transpose_bias_mkldnn_fuse_pass"); passes_.push_back("conv_elementwise_add_mkldnn_fuse_pass"); passes_.push_back("conv_concat_relu_mkldnn_fuse_pass"); - passes_.push_back("operator_activation_mkldnn_fuse_pass"); + passes_.push_back("conv_activation_mkldnn_fuse_pass"); passes_.push_back("fc_fuse_pass"); passes_.push_back("repeated_fc_relu_fuse_pass"); passes_.push_back("fc_mkldnn_pass"); passes_.push_back("fc_act_mkldnn_fuse_pass"); passes_.push_back("matmul_transpose_reshape_fuse_pass"); passes_.push_back("matmul_v2_transpose_reshape_fuse_pass"); + passes_.push_back("batch_norm_act_fuse_pass"); + passes_.push_back("softplus_activation_mkldnn_fuse_pass"); passes_.push_back("compute_propagate_scales_mkldnn_pass"); passes_.push_back("scale_matmul_fuse_pass"); passes_.push_back("reshape_transpose_matmul_mkldnn_fuse_pass"); diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h index 0639b7fbf145b7..61552e492dfa19 100644 --- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h +++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h @@ -50,28 +50,21 @@ class EltwiseMKLDNNKernel : public framework::OpKernel { private: dnnl::post_ops get_post_ops(const framework::ExecutionContext& ctx) const { dnnl::post_ops post_operations; - if (ctx.HasAttr("fuse_activation")) { - const auto fuse_activation = ctx.Attr("fuse_activation"); - const auto fuse_alpha = - ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; - const auto fuse_beta = - ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; - const auto activation_scale = ctx.HasAttr("activation_scale") - ? ctx.Attr("activation_scale") - : 1.0f; - if (fuse_activation == "hard_sigmoid") { - post_operations.append_eltwise(activation_scale, - dnnl::algorithm::eltwise_linear, - fuse_alpha, - fuse_beta); - post_operations.append_eltwise( - activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); - } else if (fuse_activation != "") { - const auto activation_algorithm = - platform::AcquireActivationAlgorithm(fuse_activation); - post_operations.append_eltwise( - activation_scale, activation_algorithm, fuse_alpha, fuse_beta); - } + if (ctx.HasAttr("activation_type")) { + const float scale = ctx.HasAttr("activation_scale") + ? ctx.Attr("activation_scale") + : 1.0f; + const float alpha = ctx.HasAttr("activation_alpha") + ? ctx.Attr("activation_alpha") + : 0.0f; + const float beta = ctx.HasAttr("activation_beta") + ? ctx.Attr("activation_beta") + : 0.0f; + + const auto activation_algorithm = platform::AcquireActivationAlgorithm( + ctx.Attr("activation_type")); + + post_operations.append_eltwise(scale, activation_algorithm, alpha, beta); } return post_operations; } diff --git a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py index f50c69b170dc3d..622d54343f6a0e 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py @@ -432,7 +432,7 @@ def _optimize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'conv_transpose_bias_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_elementwise_add_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_concat_relu_mkldnn_fuse_pass') - graph = self._apply_pass(graph, 'operator_activation_mkldnn_fuse_pass') + graph = self._apply_pass(graph, 'conv_activation_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'fc_fuse_pass', ['use_gpu', 'use_fc_padding'], [False, False]) graph = self._apply_pass(graph, 'repeated_fc_relu_fuse_pass') @@ -443,6 +443,7 @@ def _optimize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'matmul_transpose_reshape_fuse_pass') graph = self._apply_pass(graph, 'matmul_v2_transpose_reshape_fuse_pass') graph = self._apply_pass(graph, 'batch_norm_act_fuse_pass') + graph = self._apply_pass(graph, 'softplus_activation_mkldnn_fuse_pass') # the following pass should be the last one since it will work on all fused ops. graph = self._apply_pass(graph, 'runtime_context_cache_pass') return graph diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py index 8e1c93020cb63e..2a3e349a18a7d2 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py @@ -170,7 +170,7 @@ def sample_program_config(self, draw): # 11. Generate legal attr of act act_op = None - self.passes = ["operator_activation_mkldnn_fuse_pass"] + self.passes = ["conv_activation_mkldnn_fuse_pass"] if act_type == "relu6": threshold = draw(st.floats(min_value=1.0, max_value=10.0)) act_op = OpConfig("relu6", diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py index d0dbde35eb89e6..0c4bfc8ee26043 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py @@ -40,7 +40,7 @@ def setUp(self): } self.fetch_list = [conv_out] self.enable_mkldnn = True - self.pass_name = 'operator_activation_mkldnn_fuse_pass' + self.pass_name = 'conv_activation_mkldnn_fuse_pass' def set_params(self): self.conv_num_filters = 3 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py index 7019b193228c7d..96ac615f913a7f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py @@ -34,7 +34,8 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: def sample_program_config(self, draw): data_format = draw(st.sampled_from(["NCHW", "NHWC"])) dilations = draw(st.sampled_from([[1, 1], [2, 2], [1, 2]])) - padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + padding_algorithm = draw( + st.sampled_from(["EXPLICIT", "SAME", "VALID"])) groups = draw(st.sampled_from([1, 2, 4])) paddings = draw(st.sampled_from([[0, 3], [1, 2, 3, 4]])) strides = draw(st.sampled_from([[1, 1], [2, 2], [1, 2]])) @@ -103,7 +104,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["operator_activation_mkldnn_fuse_pass"]) + passes=["conv_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py index 630ddf5879e7a5..1eb325f1d08799 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py @@ -104,7 +104,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["operator_activation_mkldnn_fuse_pass"]) + passes=["conv_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py index fd788083d79e8c..a0ed73964f3770 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py @@ -106,7 +106,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["operator_activation_mkldnn_fuse_pass"]) + passes=["conv_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py index 4594cd3e86d343..6f5810cb802d9f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py @@ -100,7 +100,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["operator_activation_mkldnn_fuse_pass"]) + passes=["conv_activation_mkldnn_fuse_pass"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py index da691ccb4c6af0..12f4249a4d66df 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py @@ -25,7 +25,7 @@ class ElementwiseActivationMkldnnFusePassTest(InferencePassTest): act_alpha = None act_beta = None - pass_name = 'operator_activation_mkldnn_fuse_pass' + pass_name = 'elt_act_mkldnn_fuse_pass' def setUp(self): self.set_params() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py index a4a420e7b6a740..78393ef59b658c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py @@ -76,7 +76,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - passes=["operator_activation_mkldnn_fuse_pass"], + passes=["elt_act_mkldnn_fuse_pass"], min_success_num=4) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 089c3c6cb034cd..58d2bd3b02d00b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -1,104 +1,139 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +from __future__ import print_function +import unittest +import numpy as np +from inference_pass_test import InferencePassTest +import paddle +import paddle.fluid as fluid +from paddle.fluid.core import PassVersionChecker + + +@ @ -12, 137 + 12, 93 @ @ # See the License for the specific language governing permissions and # limitations under the License. +class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): + fuse_activation_alpha = None + fuse_activation_beta = None + pass_name = 'softplus_activation_mkldnn_fuse_pass' -from auto_scan_test import PassAutoScanTest -from program_config import TensorConfig, ProgramConfig, OpConfig -import numpy as np -from functools import partial -import unittest -import hypothesis.strategies as st - - -class TestSoftplusActivationMkldnnFusePass(PassAutoScanTest): - - def sample_program_config(self, draw): - activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', - 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', - 'hard_sigmoid', 'leaky_relu' - ])) - - def generate_input(): - shape = [16, 3, 100, 100] - return np.random.random(shape).astype(np.float32) - - softplus_op = OpConfig(type='softplus', - inputs={ - 'X': ['softplus_X'] - }, - outputs={'Out': ['softplus_output']}, - attrs={ - 'beta': 1, - 'threshold': 20, - 'use_mkldnn': True - }) - - if activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["softplus_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) - elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["softplus_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) - elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["softplus_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) - elif activation_type == "clip": - activation_op = OpConfig( - activation_type, - inputs={"X": ["softplus_output"]}, - outputs={"Out": ["activation_output"]}, - min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) - else: - activation_op = OpConfig(activation_type, - inputs={"X": ["softplus_output"]}, - outputs={"Out": ["activation_output"]}) - - model_net = [softplus_op, activation_op] - - program_config = ProgramConfig( - ops=model_net, - weights={}, - inputs={ - 'softplus_X': TensorConfig(data_gen=partial(generate_input)) - }, - outputs=['activation_output']) - - return program_config - - def sample_predictor_configs(self, program_config): - config = self.create_inference_config(use_mkldnn=True) - yield config, ['softplus'], (1e-5, 1e-5) - - def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=['operator_activation_mkldnn_fuse_pass']) - - -if __name__ == '__main__': - unittest.main() + def setUp(self): + self.set_params() + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data(name="data", + shape=[-1, 3, 100, 100], + dtype="float32") + softplus_out = fluid.layers.softplus(data) + if self.fuse_activation_beta is not None: + activation_out = self.fuse_activation( + softplus_out, self.fuse_activation_alpha, + self.fuse_activation_beta) + elif self.fuse_activation_alpha is not None: + activation_out = self.fuse_activation( + softplus_out, self.fuse_activation_alpha) + else: + activation_out = self.fuse_activation(softplus_out) + + self.feeds = { + "data": np.random.random((1, 3, 100, 100)).astype("float32"), + } + self.fetch_list = [activation_out] + self.enable_mkldnn = True + + def set_params(self): + self.fuse_activation = fluid.layers.relu + + def test_check_output(self): + use_gpu = False + self.check_output_with_option(use_gpu) + + def test_pass_compatible(self): + self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name)) + + +class SoftplusActivationTanhOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.tanh + + +class SoftplusActivationLeakyReluOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.leaky_relu + self.fuse_activation_alpha = 0.3 + + +class SoftplusActivationSwishOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.swish + self.fuse_activation_alpha = 3 + + +class SoftplusActivationHardSwishOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.hard_swish + + +class SoftplusActivationSqrtOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.hard_swish + + +class SoftplusActivationAbsOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.abs + + +class SoftplusActivationClipOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.clip + self.fuse_activation_alpha = 1.1 + self.fuse_activation_beta = 5.2 + + +class SoftplusActivationGeluErfOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.gelu + + +class SoftplusActivationGeluTanhOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.gelu + self.fuse_activation_alpha = True # simulated "Approximate" attr + + +class SoftplusActivationRelu6OneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.relu6 + + +class SoftplusActivationSigmoidOneDNNFusePassTest( + SoftplusActivationReluOneDNNFusePassTest): + + def set_params(self): + self.fuse_activation = fluid.layers.sigmoid + + +if __name__ == "__main__": + paddle.enable_static() From 0b9a000108a43f4d086a462ce9c6dc8ae2986453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 17:15:43 -0700 Subject: [PATCH 17/27] remove softmax mkldnn attrs --- .../ir/mkldnn/elt_act_mkldnn_fuse_pass.cc | 1 - .../ir/mkldnn/elt_act_mkldnn_fuse_pass.h | 46 +++++++++++++++++++ .../ir/mkldnn/fc_act_mkldnn_fuse_pass.cc | 1 - .../softplus_activation_mkldnn_fuse_pass.cc | 33 +++++++------ .../softplus_activation_mkldnn_fuse_pass.h | 2 +- .../test_mkldnn_conv_gelu_fuse_pass.py | 3 +- ...st_mkldnn_softplus_activation_fuse_pass.py | 32 ++++++------- 7 files changed, 78 insertions(+), 40 deletions(-) create mode 100644 paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc index 3867ff4ee75b85..57c7ec77a76026 100644 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc @@ -77,7 +77,6 @@ void ElementwiseActivationOneDNNPass::FuseElementwiseAct( auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse " << elt_type << " with activation op."; - GET_IR_NODE_FROM_SUBGRAPH( elementwise, preceding_op, elementwise_act_pattern); GET_IR_NODE_FROM_SUBGRAPH( diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h new file mode 100644 index 00000000000000..8df479e3ddf06d --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h @@ -0,0 +1,46 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph.h" + +namespace paddle { +namespace framework { +namespace ir { + +/* + * \brief Fuse the Elementwise and activation operators into single + * OneDNN's Elementwise with post-op. + */ +class ElementwiseActivationOneDNNPass : public FusePassBase { + public: + virtual ~ElementwiseActivationOneDNNPass() {} + + protected: + void ApplyImpl(Graph *graph) const override; + + void FuseElementwiseAct( + Graph *graph, + const std::string &elt_types, + const std::string &act_types, + const std::unordered_map &attr_map) const; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc index 34a95b207ba937..e5031c83aac160 100644 --- a/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/fc_act_mkldnn_fuse_pass.cc @@ -46,7 +46,6 @@ void FuseFCActOneDNNPass::FuseFCAct(Graph *graph, auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse fc with activation op."; - GET_IR_NODE_FROM_SUBGRAPH(fc, preceding_op, fc_act_pattern); GET_IR_NODE_FROM_SUBGRAPH(fc_out, preceding_op_out, fc_act_pattern); GET_IR_NODE_FROM_SUBGRAPH(act, activation, fc_act_pattern); diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc index d6b482714679a9..e41f81fcfbab12 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc @@ -42,15 +42,15 @@ void SoftplusActivationOneDNNPass::ApplyImpl(Graph *graph) const { std::unordered_map attr_map; if (act_type == "swish") - attr_map.emplace("beta", "fuse_activation_alpha"); + attr_map.emplace("beta", "fuse_alpha"); else if (act_type == "relu6") - attr_map.emplace("threshold", "fuse_activation_alpha"); + attr_map.emplace("threshold", "fuse_alpha"); else if (act_type == "clip") { - attr_map.emplace("min", "fuse_activation_alpha"); - attr_map.emplace("max", "fuse_activation_beta"); + attr_map.emplace("min", "fuse_alpha"); + attr_map.emplace("max", "fuse_beta"); } else { - attr_map.emplace("alpha", "fuse_activation_alpha"); - attr_map.emplace("beta", "fuse_activation_beta"); + attr_map.emplace("alpha", "fuse_alpha"); + attr_map.emplace("beta", "fuse_beta"); } FuseSoftplusActivation(graph, act_type, attr_map); } @@ -58,7 +58,7 @@ void SoftplusActivationOneDNNPass::ApplyImpl(Graph *graph) const { void SoftplusActivationOneDNNPass::FuseSoftplusActivation( Graph *graph, - const std::string &fuse_activation_type, + const std::string &fuse_activation, const std::unordered_map &attr_map) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); @@ -67,20 +67,20 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( GraphPatternDetector gpd; patterns::OperatorActivation softplus_activation_pattern( gpd.mutable_pattern(), "softplus_activation"); - softplus_activation_pattern("softplus", fuse_activation_type); + softplus_activation_pattern("softplus", fuse_activation); int found_softplus_activation_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, Graph *g) { VLOG(4) << "Fuse softplus with activation op."; - GET_IR_NODE_FROM_SUBGRAPH( - softplus, preceding_op, softplus_activation_pattern); GET_IR_NODE_FROM_SUBGRAPH( softplus_out, preceding_op_out, softplus_activation_pattern); - GET_IR_NODE_FROM_SUBGRAPH( - activation, activation, softplus_activation_pattern); GET_IR_NODE_FROM_SUBGRAPH( activation_out, activation_out, softplus_activation_pattern); + GET_IR_NODE_FROM_SUBGRAPH( + softplus, preceding_op, softplus_activation_pattern); + GET_IR_NODE_FROM_SUBGRAPH( + activation, activation, softplus_activation_pattern); auto *softplus_op = softplus->Op(); @@ -100,12 +100,11 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( } } - if (fuse_activation_type == "gelu" && - activation_op->HasAttr("approximate") && + if (fuse_activation == "gelu" && activation_op->HasAttr("approximate") && BOOST_GET_CONST(bool, activation_op->GetAttr("approximate"))) - softplus_op->SetAttr("fuse_activation_type", std::string("gelu_tanh")); + softplus_op->SetAttr("fuse_activation", std::string("gelu_tanh")); else - softplus_op->SetAttr("fuse_activation_type", fuse_activation_type); + softplus_op->SetAttr("fuse_activation", fuse_activation_type); softplus_op->SetAttr("use_mkldnn", true); @@ -121,7 +120,7 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( if (!Has("disable_logs") || !Get("disable_logs")) PrettyLogDetail("--- fused %d softplus with %s activation", found_softplus_activation_count, - fuse_activation_type); + fuse_activation); } } // namespace ir } // namespace framework diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h index 3d464a52deffbe..f36f640d3833d3 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h @@ -42,4 +42,4 @@ class SoftplusActivationOneDNNPass : public FusePassBase { } // namespace ir } // namespace framework -} // namespace paddle \ No newline at end of file +} // namespace paddle diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py index 96ac615f913a7f..dff23e96dd244d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py @@ -34,8 +34,7 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: def sample_program_config(self, draw): data_format = draw(st.sampled_from(["NCHW", "NHWC"])) dilations = draw(st.sampled_from([[1, 1], [2, 2], [1, 2]])) - padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) groups = draw(st.sampled_from([1, 2, 4])) paddings = draw(st.sampled_from([[0, 3], [1, 2, 3, 4]])) strides = draw(st.sampled_from([[1, 1], [2, 2], [1, 2]])) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 58d2bd3b02d00b..4fb2a1a5faef2e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -2,18 +2,25 @@ # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function +from paddle.fluid.core import PassVersionChecker + import unittest import numpy as np from inference_pass_test import InferencePassTest -import paddle -import paddle.fluid as fluid -from paddle.fluid.core import PassVersionChecker -@ @ -12, 137 + 12, 93 @ @ -# See the License for the specific language governing permissions and -# limitations under the License. +@ @ -22, 6 + 11, 9 @ @ class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): fuse_activation_alpha = None fuse_activation_beta = None @@ -35,7 +42,6 @@ def setUp(self): softplus_out, self.fuse_activation_alpha) else: activation_out = self.fuse_activation(softplus_out) - self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32"), } @@ -55,14 +61,12 @@ def test_pass_compatible(self): class SoftplusActivationTanhOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.tanh class SoftplusActivationLeakyReluOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.leaky_relu self.fuse_activation_alpha = 0.3 @@ -70,7 +74,6 @@ def set_params(self): class SoftplusActivationSwishOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.swish self.fuse_activation_alpha = 3 @@ -78,28 +81,24 @@ def set_params(self): class SoftplusActivationHardSwishOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationSqrtOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationAbsOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.abs class SoftplusActivationClipOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.clip self.fuse_activation_alpha = 1.1 @@ -108,14 +107,12 @@ def set_params(self): class SoftplusActivationGeluErfOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.gelu class SoftplusActivationGeluTanhOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.gelu self.fuse_activation_alpha = True # simulated "Approximate" attr @@ -123,17 +120,16 @@ def set_params(self): class SoftplusActivationRelu6OneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.relu6 class SoftplusActivationSigmoidOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): - def set_params(self): self.fuse_activation = fluid.layers.sigmoid if __name__ == "__main__": paddle.enable_static() + unittest.main() From cb4bccf060d377fde87faf5f976461a1d23bf76e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 17:43:42 -0700 Subject: [PATCH 18/27] set common name for act attributes --- paddle/fluid/framework/ir/CMakeLists.txt | 1 + .../matmul_activation_mkldnn_fuse_pass.cc | 22 +++++----- .../matmul_activation_mkldnn_fuse_pass.h | 43 +++++++++++++++++++ .../softplus_activation_mkldnn_fuse_pass.cc | 2 +- ...st_mkldnn_softplus_activation_fuse_pass.py | 17 +++++++- 5 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index d19b163817e41e..14bf83e6c0a210 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -212,6 +212,7 @@ if(WITH_MKLDNN) pass_library(shuffle_channel_mkldnn_detect_pass inference DIR mkldnn) pass_library(fc_act_mkldnn_fuse_pass inference DIR mkldnn) pass_library(elt_act_mkldnn_fuse_pass inference DIR mkldnn) + pass_library(matmul_activation_mkldnn_fuse_pass inference DIR mkldnn) pass_library(cpu_quantize_placement_pass base DIR mkldnn) pass_library(cpu_quantize_pass inference DIR mkldnn) pass_library(cpu_quantize_squash_pass inference DIR mkldnn) diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index 80a68708d51592..f01d52a508dc6f 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -45,18 +45,18 @@ void MatmulActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { std::unordered_map attrs_map; if (act_type == "swish") - attrs_map.emplace("beta", "activation_alpha"); + attrs_map.emplace("beta", "fuse_alpha"); else if (act_type == "relu6") - attrs_map.emplace("threshold", "activation_alpha"); + attrs_map.emplace("threshold", "fuse_alpha"); else if (act_type == "hard_sigmoid") { - attrs_map.emplace("slope", "activation_alpha"); - attrs_map.emplace("offset", "activation_beta"); + attrs_map.emplace("slope", "fuse_alpha"); + attrs_map.emplace("offset", "fuse_beta"); } else if (act_type == "clip") { - attrs_map.emplace("min", "activation_alpha"); - attrs_map.emplace("max", "activation_beta"); + attrs_map.emplace("min", "fuse_alpha"); + attrs_map.emplace("max", "fuse_beta"); } else { - attrs_map.emplace("alpha", "activation_alpha"); - attrs_map.emplace("beta", "activation_beta"); + attrs_map.emplace("alpha", "fuse_alpha"); + attrs_map.emplace("beta", "fuse_beta"); } FuseMatmulAct(graph, matmul_type, act_type, attrs_map); } @@ -105,10 +105,10 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( act_type = BOOST_GET_CONST(bool, activation->Op()->GetAttr("approximate")) ? "gelu_tanh" : "gelu_erf"; - matmul_op->SetAttr("activation_alpha", 0.0f); - matmul_op->SetAttr("activation_beta", 0.0f); + matmul_op->SetAttr("fuse_alpha", 0.0f); + matmul_op->SetAttr("fuse_beta", 0.0f); } - matmul_op->SetAttr("activation_type", act_type); + matmul_op->SetAttr("fuse_activation", act_type); matmul_op->SetOutput("Out", {activation_out->Name()}); IR_NODE_LINK_TO(matmul, activation_out); diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h new file mode 100644 index 00000000000000..144943e7fbb1d8 --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h @@ -0,0 +1,43 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph.h" + +namespace paddle { +namespace framework { +namespace ir { + +class MatmulActivationMkldnnFusePass : public FusePassBase { + public: + MatmulActivationMkldnnFusePass(); + virtual ~MatmulActivationMkldnnFusePass() {} + + protected: + void ApplyImpl(Graph *graph) const override; + + void FuseMatmulAct( + Graph *graph, + const std::string &matmul_type, + std::string &act_type, + const std::unordered_map &attrs_map) const; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle \ No newline at end of file diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc index e41f81fcfbab12..f4de50ed9ccf48 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc @@ -104,7 +104,7 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( BOOST_GET_CONST(bool, activation_op->GetAttr("approximate"))) softplus_op->SetAttr("fuse_activation", std::string("gelu_tanh")); else - softplus_op->SetAttr("fuse_activation", fuse_activation_type); + softplus_op->SetAttr("fuse_activation", fuse_activation); softplus_op->SetAttr("use_mkldnn", true); diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 4fb2a1a5faef2e..0c25a790138cd5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -13,14 +13,15 @@ # limitations under the License. from __future__ import print_function -from paddle.fluid.core import PassVersionChecker import unittest import numpy as np from inference_pass_test import InferencePassTest +import paddle +import paddle.fluid as fluid +from paddle.fluid.core import PassVersionChecker -@ @ -22, 6 + 11, 9 @ @ class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): fuse_activation_alpha = None fuse_activation_beta = None @@ -42,6 +43,7 @@ def setUp(self): softplus_out, self.fuse_activation_alpha) else: activation_out = self.fuse_activation(softplus_out) + self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32"), } @@ -61,12 +63,14 @@ def test_pass_compatible(self): class SoftplusActivationTanhOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.tanh class SoftplusActivationLeakyReluOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.leaky_relu self.fuse_activation_alpha = 0.3 @@ -74,6 +78,7 @@ def set_params(self): class SoftplusActivationSwishOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.swish self.fuse_activation_alpha = 3 @@ -81,24 +86,28 @@ def set_params(self): class SoftplusActivationHardSwishOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationSqrtOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationAbsOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.abs class SoftplusActivationClipOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.clip self.fuse_activation_alpha = 1.1 @@ -107,12 +116,14 @@ def set_params(self): class SoftplusActivationGeluErfOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.gelu class SoftplusActivationGeluTanhOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.gelu self.fuse_activation_alpha = True # simulated "Approximate" attr @@ -120,12 +131,14 @@ def set_params(self): class SoftplusActivationRelu6OneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.relu6 class SoftplusActivationSigmoidOneDNNFusePassTest( SoftplusActivationReluOneDNNFusePassTest): + def set_params(self): self.fuse_activation = fluid.layers.sigmoid From bab7fba85c4bce9d192c556d6d54c7646467f67e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 29 Jun 2022 17:56:56 -0700 Subject: [PATCH 19/27] whitespace --- .../framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h index 144943e7fbb1d8..0e8d613e8ae9d1 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h @@ -40,4 +40,4 @@ class MatmulActivationMkldnnFusePass : public FusePassBase { } // namespace ir } // namespace framework -} // namespace paddle \ No newline at end of file +} // namespace paddle From 36439247928484c480dbaa550a8b9656274cb841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Fri, 1 Jul 2022 09:17:26 -0700 Subject: [PATCH 20/27] append postops by helper function --- .../ir/mkldnn/elt_act_mkldnn_fuse_pass.cc | 16 ++-- .../int8_scale_calculation_mkldnn_pass.cc | 2 +- ...t8_scale_calculation_mkldnn_pass_tester.cc | 2 +- .../matmul_activation_mkldnn_fuse_pass.cc | 62 ++++++++------ .../mkldnn/elementwise_mkldnn_op.h | 17 +--- .../fluid/operators/mkldnn/conv_mkldnn_op.cc | 36 +++------ .../operators/mkldnn/matmul_mkldnn_op.cc | 22 +---- .../operators/mkldnn/softplus_mkldnn_op.h | 21 +---- paddle/fluid/platform/mkldnn_reuse.h | 81 +++++++++++++------ ...st_mkldnn_softplus_activation_fuse_pass.py | 24 +++--- 10 files changed, 129 insertions(+), 154 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc index 57c7ec77a76026..d56270c7b5a260 100644 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc @@ -45,15 +45,15 @@ void ElementwiseActivationOneDNNPass::ApplyImpl(Graph *graph) const { std::unordered_map attr_map; if (act_type == "swish") - attr_map.emplace("beta", "activation_alpha"); + attr_map.emplace("beta", "fuse_alpha"); else if (act_type == "relu6") - attr_map.emplace("threshold", "activation_alpha"); + attr_map.emplace("threshold", "fuse_alpha"); else if (act_type == "clip") { - attr_map.emplace("min", "activation_alpha"); - attr_map.emplace("max", "activation_beta"); + attr_map.emplace("min", "fuse_alpha"); + attr_map.emplace("max", "fuse_beta"); } else { - attr_map.emplace("alpha", "activation_alpha"); - attr_map.emplace("beta", "activation_beta"); + attr_map.emplace("alpha", "fuse_alpha"); + attr_map.emplace("beta", "fuse_beta"); } FuseElementwiseAct(graph, elt_type, act_type, attr_map); } @@ -108,9 +108,9 @@ void ElementwiseActivationOneDNNPass::FuseElementwiseAct( if (act_type == "gelu" && activation_op->HasAttr("approximate") && BOOST_GET_CONST(bool, activation_op->GetAttr("approximate"))) - elementwise_op->SetAttr("activation_type", std::string("gelu_tanh")); + elementwise_op->SetAttr("fuse_activation", std::string("gelu_tanh")); else - elementwise_op->SetAttr("activation_type", act_type); + elementwise_op->SetAttr("fuse_activation", act_type); elementwise_op->SetOutput("Out", {activation_out->Name()}); diff --git a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc index a5481f5c6f30e0..b505e3394628bb 100644 --- a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc @@ -156,7 +156,7 @@ void Int8ScaleCalculationMkldnnPass::ApplyImpl(ir::Graph* graph) const { conv_op->Op()->SetAttr("Sum_scale", sum_scale); conv_op->Op()->SetAttr("Output_shift_scale", output_shift_scale); - conv_op->Op()->SetAttr("Activation_scale", activation_scale); + conv_op->Op()->SetAttr("activation_scale", activation_scale); found_int8_scales_count++; }; gpd(graph, handler); diff --git a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc index fd6c7d20a95d2e..d985b3f9083b01 100644 --- a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc @@ -123,7 +123,7 @@ void MainTest(bool convWithExistingBias, EXPECT_EQ( op->GetAttrIfExists>("Output_shift_scale")[0], scale / scale_weights[0]); - EXPECT_EQ(op->GetAttrIfExists("Activation_scale"), scale); + EXPECT_EQ(op->GetAttrIfExists("activation_scale"), scale); if (convWithExistingBias) { EXPECT_EQ(op->GetAttrIfExists>("Bias_scales")[0], diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index f01d52a508dc6f..68651cef2b359d 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -146,7 +146,7 @@ MatmulActivationMkldnnFusePass::MatmulActivationMkldnnFusePass() { .IsType() .End(); - AddOpCompat(OpCompat("relu")) + AddOpCompat(OpCompat("abs")) .AddInput("X") .IsTensor() .End() @@ -154,36 +154,42 @@ MatmulActivationMkldnnFusePass::MatmulActivationMkldnnFusePass() { .IsTensor() .End(); - AddOpCompat(OpCompat("leaky_relu")) + AddOpCompat(OpCompat("clip")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() .End() - .AddAttr("alpha") - .IsType() + .AddAttr("min") + .End() + .AddAttr("max") .End(); - AddOpCompat(OpCompat("relu6")) + AddOpCompat(OpCompat("gelu")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() .End() - .AddAttr("threshold") - .IsType() + .AddAttr("approximate") + .IsType() .End(); - AddOpCompat(OpCompat("swish")) + AddOpCompat(OpCompat("hard_sigmoid")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() .End() - .AddAttr("beta") + .AddAttr("slope") + .IsOptional() + .IsType() + .End() + .AddAttr("offset") + .IsOptional() .IsType() .End(); @@ -207,47 +213,42 @@ MatmulActivationMkldnnFusePass::MatmulActivationMkldnnFusePass() { .IsType() .End(); - AddOpCompat(OpCompat("mish")) + AddOpCompat(OpCompat("leaky_relu")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() + .End() + .AddAttr("alpha") + .IsType() .End(); - AddOpCompat(OpCompat("hard_sigmoid")) + AddOpCompat(OpCompat("mish")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() - .End() - .AddAttr("slope") - .IsOptional() - .IsType() - .End() - .AddAttr("offset") - .IsOptional() - .IsType() .End(); - AddOpCompat(OpCompat("gelu")) + AddOpCompat(OpCompat("relu")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() - .End() - .AddAttr("approximate") - .IsType() .End(); - AddOpCompat(OpCompat("tanh")) + AddOpCompat(OpCompat("relu6")) .AddInput("X") .IsTensor() .End() .AddOutput("Out") .IsTensor() + .End() + .AddAttr("threshold") + .IsType() .End(); AddOpCompat(OpCompat("sigmoid")) @@ -266,7 +267,18 @@ MatmulActivationMkldnnFusePass::MatmulActivationMkldnnFusePass() { .IsTensor() .End(); - AddOpCompat(OpCompat("abs")) + AddOpCompat(OpCompat("swish")) + .AddInput("X") + .IsTensor() + .End() + .AddOutput("Out") + .IsTensor() + .End() + .AddAttr("beta") + .IsType() + .End(); + + AddOpCompat(OpCompat("tanh")) .AddInput("X") .IsTensor() .End() diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h index 61552e492dfa19..b96efa86c3d9e3 100644 --- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h +++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h @@ -50,21 +50,8 @@ class EltwiseMKLDNNKernel : public framework::OpKernel { private: dnnl::post_ops get_post_ops(const framework::ExecutionContext& ctx) const { dnnl::post_ops post_operations; - if (ctx.HasAttr("activation_type")) { - const float scale = ctx.HasAttr("activation_scale") - ? ctx.Attr("activation_scale") - : 1.0f; - const float alpha = ctx.HasAttr("activation_alpha") - ? ctx.Attr("activation_alpha") - : 0.0f; - const float beta = ctx.HasAttr("activation_beta") - ? ctx.Attr("activation_beta") - : 0.0f; - - const auto activation_algorithm = platform::AcquireActivationAlgorithm( - ctx.Attr("activation_type")); - - post_operations.append_eltwise(scale, activation_algorithm, alpha, beta); + if (ctx.HasAttr("fuse_activation")) { + platform::AppendActivation(ctx, post_operations); } return post_operations; } diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc index 17d4c2fad96b8e..26e542b07f76c5 100644 --- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc @@ -504,7 +504,7 @@ class ConvMKLDNNHandlerT return bias_scale_tuple; } - std::tuple, float> get_int8_scales( + std::tuple> get_int8_scales( const framework::ExecutionContext& ctx) const { const auto* filter = ctx.Input("Filter"); const auto& weights_tz = phi::vectorize(filter->dims()); @@ -518,9 +518,6 @@ class ConvMKLDNNHandlerT auto scale_weights_data = ctx.Attr>("Scale_weights"); bool is_multi_channel = scale_weights_data.size() > 1; bool has_activation = !ctx.Attr("fuse_activation").empty(); - float activation_scale = (!force_fp32_output && has_activation) - ? ctx.Attr("Scale_out") - : 1.0f; float scale_out_data = (force_fp32_output || has_activation) ? 1.0f @@ -546,30 +543,23 @@ class ConvMKLDNNHandlerT static_cast(scale_weights_data[i]))); } - return std::make_tuple(sum_scale, output_shift_scale, activation_scale); + return std::make_tuple(sum_scale, output_shift_scale); } dnnl::primitive_attr CreateConvAttrs(const framework::ExecutionContext& ctx) { dnnl::primitive_attr conv_attr; dnnl::post_ops post_operations; - const std::string fuse_activation = - ctx.Attr("fuse_activation"); - const float fuse_alpha = ctx.Attr("fuse_alpha"); - const float fuse_beta = ctx.Attr("fuse_beta"); const bool fuse_residual_conn = ctx.Attr("fuse_residual_connection"); float sum_scale = 1.0f; - float activation_scale = 1.0f; std::vector output_shift_scale; if (platform::is_int8()) { if (ctx.HasAttr("Sum_scale")) { sum_scale = ctx.Attr("Sum_scale"); - activation_scale = ctx.Attr("Activation_scale"); output_shift_scale = ctx.Attr>("Output_shift_scale"); } else { - std::tie(sum_scale, output_shift_scale, activation_scale) = - get_int8_scales(ctx); + std::tie(sum_scale, output_shift_scale) = get_int8_scales(ctx); } if (output_shift_scale.size() > 0) { @@ -587,18 +577,14 @@ class ConvMKLDNNHandlerT post_operations.append_sum(sum_scale); } - if (fuse_activation == "hard_sigmoid") { - post_operations.append_eltwise(activation_scale, - dnnl::algorithm::eltwise_linear, - fuse_alpha, - fuse_beta); - post_operations.append_eltwise( - activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); - } else if (fuse_activation != "") { - const auto activation_algorithm = - platform::AcquireActivationAlgorithm(fuse_activation); - post_operations.append_eltwise( - activation_scale, activation_algorithm, fuse_alpha, fuse_beta); + if (!ctx.Attr("fuse_activation").empty()) { + float activation_scale = 1.0f; + if (platform::is_int8() && !ctx.HasAttr("Sum_scale") && + !ctx.Attr("force_fp32_output")) { + activation_scale = ctx.Attr("Scale_out"); + } + paddle::platform::AppendActivation( + ctx, post_operations, activation_scale); } conv_attr.set_post_ops(post_operations); diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index 30214b6b2febc4..59952bf71bea81 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -455,27 +455,7 @@ class MatMulMKLDNNHandler } if (ctx.HasAttr("fuse_activation")) { - const auto fuse_activation = ctx.Attr("fuse_activation"); - const auto fuse_alpha = - ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; - const auto fuse_beta = - ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; - const auto activation_scale = ctx.HasAttr("activation_scale") - ? ctx.Attr("activation_scale") - : 1.0f; - if (fuse_activation == "hard_sigmoid") { - post_operations.append_eltwise(activation_scale, - dnnl::algorithm::eltwise_linear, - fuse_alpha, - fuse_beta); - post_operations.append_eltwise( - activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); - } else if (fuse_activation != "") { - const auto activation_algorithm = - paddle::platform::AcquireActivationAlgorithm(fuse_activation); - post_operations.append_eltwise( - activation_scale, activation_algorithm, fuse_alpha, fuse_beta); - } + paddle::platform::AppendActivation(ctx, post_operations); } matmul_attrs.set_post_ops(post_operations); diff --git a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h index c947d63c64203a..09c2f4efbc27d7 100644 --- a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h +++ b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h @@ -47,26 +47,7 @@ class SoftplusMKLDNNHandler } if (ctx.HasAttr("fuse_activation")) { - const auto fuse_activation = ctx.Attr("fuse_activation"); - const auto fuse_alpha = - ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; - const auto fuse_beta = - ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; - const auto activation_scale = 1.0f; - - if (fuse_activation == "hard_sigmoid") { - post_ops.append_eltwise(activation_scale, - dnnl::algorithm::eltwise_linear, - fuse_alpha, - fuse_beta); - post_ops.append_eltwise( - activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); - } else if (fuse_activation != "") { - const auto activation_algorithm = - platform::AcquireActivationAlgorithm(fuse_activation); - post_ops.append_eltwise( - activation_scale, activation_algorithm, fuse_alpha, fuse_beta); - } + platform::AppendActivation(ctx, post_ops); } dnnl::primitive_attr attrs; diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index 05ebedf611a4b2..27b9ffe4b16cc4 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -1009,32 +1009,61 @@ class ActivationMKLDNNHandler } }; -static const dnnl::algorithm AcquireActivationAlgorithm( - std::string activation_name) { - std::unordered_map activation_map = { - {"abs", dnnl::algorithm::eltwise_abs}, - {"clip", dnnl::algorithm::eltwise_clip}, - {"gelu", dnnl::algorithm::eltwise_gelu_erf}, - {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf}, - {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh}, - {"hard_swish", dnnl::algorithm::eltwise_hardswish}, - {"leaky_relu", dnnl::algorithm::eltwise_relu}, - {"mish", dnnl::algorithm::eltwise_mish}, - {"relu", dnnl::algorithm::eltwise_relu}, - {"relu6", dnnl::algorithm::eltwise_bounded_relu}, - {"sigmoid", dnnl::algorithm::eltwise_logistic}, - {"sqrt", dnnl::algorithm::eltwise_sqrt}, - {"swish", dnnl::algorithm::eltwise_swish}, - {"tanh", dnnl::algorithm::eltwise_tanh}}; - - const auto& activation_type = activation_map.find(activation_name); - - PADDLE_ENFORCE_NE(activation_type, - activation_map.end(), - platform::errors::InvalidArgument( - "Activation '%s' not found in oneDNN algorithms mapper", - activation_name)); - return activation_type->second; +static void AppendActivation(const framework::ExecutionContext& ctx, + dnnl::post_ops& post_ops, + float quantized_activation_scale = 1.0f) { + const auto fuse_activation = ctx.Attr("fuse_activation"); + const auto fuse_alpha = + ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; + const auto fuse_beta = + ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; + + float activation_scale; + + if (quantized_activation_scale != 1.0f) { + activation_scale = quantized_activation_scale; + } else { + activation_scale = ctx.HasAttr("activation_scale") + ? ctx.Attr("activation_scale") + : 1.0f; + } + + if (fuse_activation == "hard_sigmoid") { + post_ops.append_eltwise(activation_scale, + dnnl::algorithm::eltwise_linear, + fuse_alpha, + fuse_beta); + post_ops.append_eltwise( + activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f); + } else { + const std::unordered_map activation_map = { + {"abs", dnnl::algorithm::eltwise_abs}, + {"clip", dnnl::algorithm::eltwise_clip}, + {"gelu", dnnl::algorithm::eltwise_gelu_erf}, + {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf}, + {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh}, + {"hard_swish", dnnl::algorithm::eltwise_hardswish}, + {"leaky_relu", dnnl::algorithm::eltwise_relu}, + {"mish", dnnl::algorithm::eltwise_mish}, + {"relu", dnnl::algorithm::eltwise_relu}, + {"relu6", dnnl::algorithm::eltwise_bounded_relu}, + {"sigmoid", dnnl::algorithm::eltwise_logistic}, + {"sqrt", dnnl::algorithm::eltwise_sqrt}, + {"swish", dnnl::algorithm::eltwise_swish}, + {"tanh", dnnl::algorithm::eltwise_tanh}}; + + const auto& activation_type = activation_map.find(fuse_activation); + + PADDLE_ENFORCE_NE( + activation_type, + activation_map.end(), + platform::errors::InvalidArgument( + "Activation '%s' not found in oneDNN algorithms mapper", + fuse_activation)); + + post_ops.append_eltwise( + activation_scale, activation_type->second, fuse_alpha, fuse_beta); + } } class ReorderMKLDNNHandler { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 0c25a790138cd5..cc98045b41c509 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -23,8 +23,8 @@ class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): - fuse_activation_alpha = None - fuse_activation_beta = None + fuse_alpha = None + fuse_beta = None pass_name = 'softplus_activation_mkldnn_fuse_pass' def setUp(self): @@ -34,13 +34,13 @@ def setUp(self): shape=[-1, 3, 100, 100], dtype="float32") softplus_out = fluid.layers.softplus(data) - if self.fuse_activation_beta is not None: + if self.fuse_beta is not None: activation_out = self.fuse_activation( - softplus_out, self.fuse_activation_alpha, - self.fuse_activation_beta) - elif self.fuse_activation_alpha is not None: + softplus_out, self.fuse_alpha, + self.fuse_beta) + elif self.fuse_alpha is not None: activation_out = self.fuse_activation( - softplus_out, self.fuse_activation_alpha) + softplus_out, self.fuse_alpha) else: activation_out = self.fuse_activation(softplus_out) @@ -73,7 +73,7 @@ class SoftplusActivationLeakyReluOneDNNFusePassTest( def set_params(self): self.fuse_activation = fluid.layers.leaky_relu - self.fuse_activation_alpha = 0.3 + self.fuse_alpha = 0.3 class SoftplusActivationSwishOneDNNFusePassTest( @@ -81,7 +81,7 @@ class SoftplusActivationSwishOneDNNFusePassTest( def set_params(self): self.fuse_activation = fluid.layers.swish - self.fuse_activation_alpha = 3 + self.fuse_alpha = 3 class SoftplusActivationHardSwishOneDNNFusePassTest( @@ -110,8 +110,8 @@ class SoftplusActivationClipOneDNNFusePassTest( def set_params(self): self.fuse_activation = fluid.layers.clip - self.fuse_activation_alpha = 1.1 - self.fuse_activation_beta = 5.2 + self.fuse_alpha = 1.1 + self.fuse_beta = 5.2 class SoftplusActivationGeluErfOneDNNFusePassTest( @@ -126,7 +126,7 @@ class SoftplusActivationGeluTanhOneDNNFusePassTest( def set_params(self): self.fuse_activation = fluid.layers.gelu - self.fuse_activation_alpha = True # simulated "Approximate" attr + self.fuse_alpha = True # simulated "Approximate" attr class SoftplusActivationRelu6OneDNNFusePassTest( From e29898da611309de5393edd98f131eb145c71550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Fri, 1 Jul 2022 09:31:34 -0700 Subject: [PATCH 21/27] ut style --- .../test_mkldnn_softplus_activation_fuse_pass.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index cc98045b41c509..5e5dd4c719d98b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -35,12 +35,12 @@ def setUp(self): dtype="float32") softplus_out = fluid.layers.softplus(data) if self.fuse_beta is not None: - activation_out = self.fuse_activation( - softplus_out, self.fuse_alpha, - self.fuse_beta) + activation_out = self.fuse_activation(softplus_out, + self.fuse_alpha, + self.fuse_beta) elif self.fuse_alpha is not None: - activation_out = self.fuse_activation( - softplus_out, self.fuse_alpha) + activation_out = self.fuse_activation(softplus_out, + self.fuse_alpha) else: activation_out = self.fuse_activation(softplus_out) From f9c6b0cb44a5841df76c07004ae70763d05a6d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Sun, 3 Jul 2022 06:01:25 -0700 Subject: [PATCH 22/27] revert changes related to quantization --- .../int8_scale_calculation_mkldnn_pass.cc | 2 +- ...int8_scale_calculation_mkldnn_pass_tester.cc | 2 +- paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc | 17 +++++++++-------- paddle/fluid/platform/mkldnn_reuse.h | 12 +----------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc index b505e3394628bb..a5481f5c6f30e0 100644 --- a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass.cc @@ -156,7 +156,7 @@ void Int8ScaleCalculationMkldnnPass::ApplyImpl(ir::Graph* graph) const { conv_op->Op()->SetAttr("Sum_scale", sum_scale); conv_op->Op()->SetAttr("Output_shift_scale", output_shift_scale); - conv_op->Op()->SetAttr("activation_scale", activation_scale); + conv_op->Op()->SetAttr("Activation_scale", activation_scale); found_int8_scales_count++; }; gpd(graph, handler); diff --git a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc index d985b3f9083b01..fd6c7d20a95d2e 100644 --- a/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc +++ b/paddle/fluid/framework/ir/mkldnn/int8_scale_calculation_mkldnn_pass_tester.cc @@ -123,7 +123,7 @@ void MainTest(bool convWithExistingBias, EXPECT_EQ( op->GetAttrIfExists>("Output_shift_scale")[0], scale / scale_weights[0]); - EXPECT_EQ(op->GetAttrIfExists("activation_scale"), scale); + EXPECT_EQ(op->GetAttrIfExists("Activation_scale"), scale); if (convWithExistingBias) { EXPECT_EQ(op->GetAttrIfExists>("Bias_scales")[0], diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc index 26e542b07f76c5..03a6bc71b85562 100644 --- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc @@ -504,7 +504,7 @@ class ConvMKLDNNHandlerT return bias_scale_tuple; } - std::tuple> get_int8_scales( + std::tuple, float> get_int8_scales( const framework::ExecutionContext& ctx) const { const auto* filter = ctx.Input("Filter"); const auto& weights_tz = phi::vectorize(filter->dims()); @@ -518,6 +518,9 @@ class ConvMKLDNNHandlerT auto scale_weights_data = ctx.Attr>("Scale_weights"); bool is_multi_channel = scale_weights_data.size() > 1; bool has_activation = !ctx.Attr("fuse_activation").empty(); + float activation_scale = (!force_fp32_output && has_activation) + ? ctx.Attr("Scale_out") + : 1.0f; float scale_out_data = (force_fp32_output || has_activation) ? 1.0f @@ -543,7 +546,7 @@ class ConvMKLDNNHandlerT static_cast(scale_weights_data[i]))); } - return std::make_tuple(sum_scale, output_shift_scale); + return std::make_tuple(sum_scale, output_shift_scale, activation_scale); } dnnl::primitive_attr CreateConvAttrs(const framework::ExecutionContext& ctx) { @@ -553,13 +556,16 @@ class ConvMKLDNNHandlerT const bool fuse_residual_conn = ctx.Attr("fuse_residual_connection"); float sum_scale = 1.0f; + float activation_scale = 1.0f; std::vector output_shift_scale; if (platform::is_int8()) { if (ctx.HasAttr("Sum_scale")) { sum_scale = ctx.Attr("Sum_scale"); + activation_scale = ctx.Attr("Activation_scale"); output_shift_scale = ctx.Attr>("Output_shift_scale"); } else { - std::tie(sum_scale, output_shift_scale) = get_int8_scales(ctx); + std::tie(sum_scale, output_shift_scale, activation_scale) = + get_int8_scales(ctx); } if (output_shift_scale.size() > 0) { @@ -578,11 +584,6 @@ class ConvMKLDNNHandlerT } if (!ctx.Attr("fuse_activation").empty()) { - float activation_scale = 1.0f; - if (platform::is_int8() && !ctx.HasAttr("Sum_scale") && - !ctx.Attr("force_fp32_output")) { - activation_scale = ctx.Attr("Scale_out"); - } paddle::platform::AppendActivation( ctx, post_operations, activation_scale); } diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index 27b9ffe4b16cc4..731405a18b3d3b 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -1011,23 +1011,13 @@ class ActivationMKLDNNHandler static void AppendActivation(const framework::ExecutionContext& ctx, dnnl::post_ops& post_ops, - float quantized_activation_scale = 1.0f) { + float activation_scale = 1.0f) { const auto fuse_activation = ctx.Attr("fuse_activation"); const auto fuse_alpha = ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; const auto fuse_beta = ctx.HasAttr("fuse_beta") ? ctx.Attr("fuse_beta") : 0.0f; - float activation_scale; - - if (quantized_activation_scale != 1.0f) { - activation_scale = quantized_activation_scale; - } else { - activation_scale = ctx.HasAttr("activation_scale") - ? ctx.Attr("activation_scale") - : 1.0f; - } - if (fuse_activation == "hard_sigmoid") { post_ops.append_eltwise(activation_scale, dnnl::algorithm::eltwise_linear, From 942ba41cb27f0f46ae267d8c7a3c2bbdff12e759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Mon, 4 Jul 2022 07:21:14 -0700 Subject: [PATCH 23/27] Reduce redundancy --- .../conv_activation_mkldnn_fuse_pass.cc | 41 +++------------ .../ir/mkldnn/elt_act_mkldnn_fuse_pass.cc | 51 +++++++------------ .../matmul_activation_mkldnn_fuse_pass.cc | 43 +++------------- .../softplus_activation_mkldnn_fuse_pass.cc | 48 ++++++----------- .../mkldnn/elementwise_mkldnn_op.h | 4 +- .../fluid/operators/mkldnn/conv_mkldnn_op.cc | 3 +- .../operators/mkldnn/matmul_mkldnn_op.cc | 4 +- .../operators/mkldnn/softplus_mkldnn_op.h | 4 +- paddle/fluid/platform/mkldnn_reuse.h | 37 ++++++++++++++ 9 files changed, 91 insertions(+), 144 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc index 8cf31239399539..913bb84255ed57 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc @@ -15,50 +15,25 @@ #include "paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h" #include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" #include "paddle/fluid/string/pretty_log.h" namespace paddle { namespace framework { namespace ir { +using paddle::platform::GetAttributeMap; +using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void ConvActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - std::vector act_types = {"relu", - "mish", - "swish", - "sqrt", - "hard_swish", - "sigmoid", - "abs", - "gelu", - "relu6", - "clip", - "tanh", - "hard_sigmoid", - "leaky_relu"}; - + auto act_types = GetSupportedActivations(); std::vector conv_types = {"conv2d"}; for (const auto& conv_type : conv_types) for (auto& act_type : act_types) { - std::unordered_map attrs_map; - - if (act_type == "swish") - attrs_map.emplace("beta", "fuse_alpha"); - else if (act_type == "relu6") - attrs_map.emplace("threshold", "fuse_alpha"); - else if (act_type == "hard_sigmoid") { - attrs_map.emplace("slope", "fuse_alpha"); - attrs_map.emplace("offset", "fuse_beta"); - } else if (act_type == "clip") { - attrs_map.emplace("min", "fuse_alpha"); - attrs_map.emplace("max", "fuse_beta"); - } else { - attrs_map.emplace("alpha", "fuse_alpha"); - attrs_map.emplace("beta", "fuse_beta"); - } - FuseConvAct(graph, conv_type, act_type, attrs_map); + auto attr_map = GetAttributeMap(act_type); + FuseConvAct(graph, conv_type, act_type, attr_map); } } @@ -66,7 +41,7 @@ void ConvActivationMkldnnFusePass::FuseConvAct( Graph* graph, const std::string& conv_type, std::string& act_type, - const std::unordered_map& attrs_map) const { + const std::unordered_map& attr_map) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); FusePassBase::Init(conv_type + "_" + act_type + "_mkldnn_fuse_pass", graph); @@ -94,7 +69,7 @@ void ConvActivationMkldnnFusePass::FuseConvAct( OpDesc* conv_op = conv->Op(); OpDesc* act_op = activation->Op(); - for (const auto& attrs : attrs_map) { + for (const auto& attrs : attr_map) { if (act_op->HasAttr(attrs.first)) { conv_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); } diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc index d56270c7b5a260..85a1de4357abc8 100644 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc @@ -17,44 +17,25 @@ #include "paddle/fluid/framework/ir/graph_pattern_detector.h" #include "paddle/fluid/framework/op_version_registry.h" #include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" #include "paddle/fluid/string/pretty_log.h" namespace paddle { namespace framework { namespace ir { +using paddle::platform::GetAttributeMap; +using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void ElementwiseActivationOneDNNPass::ApplyImpl(Graph *graph) const { - std::vector act_types = {"relu", - "tanh", - "leaky_relu", - "swish", - "hard_swish", - "sqrt", - "abs", - "clip", - "gelu", - "relu6", - "sigmoid"}; + auto act_types = GetSupportedActivations(); std::vector elt_types = { "elementwise_add", "elementwise_sub", "elementwise_mul"}; for (const auto &elt_type : elt_types) for (const auto &act_type : act_types) { - std::unordered_map attr_map; - - if (act_type == "swish") - attr_map.emplace("beta", "fuse_alpha"); - else if (act_type == "relu6") - attr_map.emplace("threshold", "fuse_alpha"); - else if (act_type == "clip") { - attr_map.emplace("min", "fuse_alpha"); - attr_map.emplace("max", "fuse_beta"); - } else { - attr_map.emplace("alpha", "fuse_alpha"); - attr_map.emplace("beta", "fuse_beta"); - } + auto attr_map = GetAttributeMap(act_type); FuseElementwiseAct(graph, elt_type, act_type, attr_map); } } @@ -139,14 +120,16 @@ REGISTER_PASS_CAPABILITY(elt_act_mkldnn_fuse_pass) .LE("elementwise_add", 1) .LE("elementwise_sub", 1) .LE("elementwise_mul", 1) - .LE("relu", 0) - .LE("tanh", 0) - .LE("leaky_relu", 1) - .LE("swish", 0) - .LE("hard_swish", 0) - .LE("sqrt", 0) - .LE("abs", 0) + .EQ("abs", 0) .LE("clip", 1) - .LE("gelu", 0) - .LE("relu6", 0) - .LE("sigmoid", 0)); + .EQ("gelu", 0) + .EQ("hard_sigmoid", 0) + .LE("hard_swish", 0) + .LE("leaky_relu", 1) + .LE("mish", 1) + .EQ("relu", 0) + .EQ("relu6", 0) + .EQ("sigmoid", 0) + .EQ("sqrt", 0) + .EQ("swish", 0) + .EQ("tanh", 0)); diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index 68651cef2b359d..1996c11710ccf7 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -15,50 +15,25 @@ #include "paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h" #include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" #include "paddle/fluid/string/pretty_log.h" namespace paddle { namespace framework { namespace ir { +using paddle::platform::GetAttributeMap; +using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void MatmulActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - std::vector act_types = {"abs", - "clip", - "gelu", - "hard_sigmoid", - "hard_swish", - "leaky_relu", - "mish", - "relu", - "relu6", - "sigmoid", - "sqrt", - "swish", - "tanh"}; - + auto act_types = GetSupportedActivations(); std::vector matmul_types = {"matmul"}; for (const auto& matmul_type : matmul_types) for (auto& act_type : act_types) { - std::unordered_map attrs_map; - - if (act_type == "swish") - attrs_map.emplace("beta", "fuse_alpha"); - else if (act_type == "relu6") - attrs_map.emplace("threshold", "fuse_alpha"); - else if (act_type == "hard_sigmoid") { - attrs_map.emplace("slope", "fuse_alpha"); - attrs_map.emplace("offset", "fuse_beta"); - } else if (act_type == "clip") { - attrs_map.emplace("min", "fuse_alpha"); - attrs_map.emplace("max", "fuse_beta"); - } else { - attrs_map.emplace("alpha", "fuse_alpha"); - attrs_map.emplace("beta", "fuse_beta"); - } - FuseMatmulAct(graph, matmul_type, act_type, attrs_map); + auto attr_map = GetAttributeMap(act_type); + FuseMatmulAct(graph, matmul_type, act_type, attr_map); } } @@ -66,7 +41,7 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( Graph* graph, const std::string& matmul_type, std::string& act_type, - const std::unordered_map& attrs_map) const { + const std::unordered_map& attr_map) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); FusePassBase::Init(matmul_type + "_" + act_type + "_mkldnn_fuse_pass", graph); @@ -95,7 +70,7 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( OpDesc* matmul_op = matmul->Op(); OpDesc* act_op = activation->Op(); - for (const auto& attrs : attrs_map) { + for (const auto& attrs : attr_map) { if (act_op->HasAttr(attrs.first)) { matmul_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); } @@ -105,8 +80,6 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( act_type = BOOST_GET_CONST(bool, activation->Op()->GetAttr("approximate")) ? "gelu_tanh" : "gelu_erf"; - matmul_op->SetAttr("fuse_alpha", 0.0f); - matmul_op->SetAttr("fuse_beta", 0.0f); } matmul_op->SetAttr("fuse_activation", act_type); matmul_op->SetOutput("Out", {activation_out->Name()}); diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc index f4de50ed9ccf48..729b2327684107 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc @@ -17,41 +17,22 @@ #include "paddle/fluid/framework/ir/graph_pattern_detector.h" #include "paddle/fluid/framework/op_version_registry.h" #include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" #include "paddle/fluid/string/pretty_log.h" namespace paddle { namespace framework { namespace ir { +using paddle::platform::GetAttributeMap; +using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void SoftplusActivationOneDNNPass::ApplyImpl(Graph *graph) const { - std::vector act_types = {"relu", - "tanh", - "leaky_relu", - "swish", - "hardswish", - "sqrt", - "abs", - "clip", - "gelu", - "relu6", - "sigmoid"}; + auto act_types = GetSupportedActivations(); for (const auto &act_type : act_types) { - std::unordered_map attr_map; - - if (act_type == "swish") - attr_map.emplace("beta", "fuse_alpha"); - else if (act_type == "relu6") - attr_map.emplace("threshold", "fuse_alpha"); - else if (act_type == "clip") { - attr_map.emplace("min", "fuse_alpha"); - attr_map.emplace("max", "fuse_beta"); - } else { - attr_map.emplace("alpha", "fuse_alpha"); - attr_map.emplace("beta", "fuse_beta"); - } + auto attr_map = GetAttributeMap(act_type); FuseSoftplusActivation(graph, act_type, attr_map); } } @@ -132,13 +113,16 @@ REGISTER_PASS_CAPABILITY(softplus_activation_mkldnn_fuse_pass) .AddCombination( paddle::framework::compatible::OpVersionComparatorCombination() .LE("softplus", 1) - .EQ("relu", 0) - .EQ("tanh", 0) - .LE("leaky_relu", 1) - .EQ("swish", 0) - .EQ("hard_swish", 0) - .EQ("sqrt", 0) .EQ("abs", 0) - .LE("relu6", 1) .LE("clip", 1) - .EQ("gelu", 0)); + .EQ("gelu", 0) + .EQ("hard_sigmoid", 0) + .LE("hard_swish", 0) + .LE("leaky_relu", 1) + .LE("mish", 1) + .EQ("relu", 0) + .EQ("relu6", 0) + .EQ("sigmoid", 0) + .EQ("sqrt", 0) + .EQ("swish", 0) + .EQ("tanh", 0)); diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h index b96efa86c3d9e3..615cbcc458378f 100644 --- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h +++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h @@ -50,9 +50,7 @@ class EltwiseMKLDNNKernel : public framework::OpKernel { private: dnnl::post_ops get_post_ops(const framework::ExecutionContext& ctx) const { dnnl::post_ops post_operations; - if (ctx.HasAttr("fuse_activation")) { - platform::AppendActivation(ctx, post_operations); - } + platform::AppendActivation(ctx, post_operations); return post_operations; } diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc index 03a6bc71b85562..f676dca7d64ae7 100644 --- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc @@ -583,7 +583,8 @@ class ConvMKLDNNHandlerT post_operations.append_sum(sum_scale); } - if (!ctx.Attr("fuse_activation").empty()) { + bool has_activation = !ctx.Attr("fuse_activation").empty(); + if (has_activation) { paddle::platform::AppendActivation( ctx, post_operations, activation_scale); } diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc index 59952bf71bea81..912b1be813a58f 100644 --- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc @@ -454,9 +454,7 @@ class MatMulMKLDNNHandler matmul_attrs.set_output_scales(0, {scale_out}); } - if (ctx.HasAttr("fuse_activation")) { - paddle::platform::AppendActivation(ctx, post_operations); - } + paddle::platform::AppendActivation(ctx, post_operations); matmul_attrs.set_post_ops(post_operations); return matmul_attrs; diff --git a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h index 09c2f4efbc27d7..c41864ee26f55b 100644 --- a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h +++ b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h @@ -46,9 +46,7 @@ class SoftplusMKLDNNHandler 1.0f, dnnl::algorithm::eltwise_linear, 1.0f / beta, 0.0f); } - if (ctx.HasAttr("fuse_activation")) { - platform::AppendActivation(ctx, post_ops); - } + platform::AppendActivation(ctx, post_ops); dnnl::primitive_attr attrs; attrs.set_post_ops(post_ops); diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index 731405a18b3d3b..68e6fcab160cb8 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -1012,6 +1012,7 @@ class ActivationMKLDNNHandler static void AppendActivation(const framework::ExecutionContext& ctx, dnnl::post_ops& post_ops, float activation_scale = 1.0f) { + if (!ctx.HasAttr("fuse_activation")) return; const auto fuse_activation = ctx.Attr("fuse_activation"); const auto fuse_alpha = ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; @@ -1056,6 +1057,42 @@ static void AppendActivation(const framework::ExecutionContext& ctx, } } +static std::unordered_map GetAttributeMap( + std::string act_type) { + std::unordered_map attr_map; + if (act_type == "swish") + attr_map.emplace("beta", "fuse_alpha"); + else if (act_type == "relu6") + attr_map.emplace("threshold", "fuse_alpha"); + else if (act_type == "hard_sigmoid") { + attr_map.emplace("slope", "fuse_alpha"); + attr_map.emplace("offset", "fuse_beta"); + } else if (act_type == "clip") { + attr_map.emplace("min", "fuse_alpha"); + attr_map.emplace("max", "fuse_beta"); + } else { + attr_map.emplace("alpha", "fuse_alpha"); + attr_map.emplace("beta", "fuse_beta"); + } + return attr_map; +} + +static std::vector GetSupportedActivations() { + return std::vector{"abs", + "clip", + "gelu", + "hard_sigmoid", + "hard_swish", + "leaky_relu", + "mish", + "relu", + "relu6", + "sigmoid", + "sqrt", + "swish", + "tanh"}; +} + class ReorderMKLDNNHandler { public: ReorderMKLDNNHandler(std::vector& dims, // NOLINT From 725161e94c8ed1b874be1f659992cf1e3d4daf6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 5 Jul 2022 01:24:10 -0700 Subject: [PATCH 24/27] reduce number of parameters --- .../conv_activation_mkldnn_fuse_pass.cc | 16 ++++++--------- .../mkldnn/conv_activation_mkldnn_fuse_pass.h | 8 +++----- .../ir/mkldnn/elt_act_mkldnn_fuse_pass.cc | 11 ++++------ .../ir/mkldnn/elt_act_mkldnn_fuse_pass.h | 8 +++----- .../matmul_activation_mkldnn_fuse_pass.cc | 13 ++++-------- .../matmul_activation_mkldnn_fuse_pass.h | 8 +++----- .../softplus_activation_mkldnn_fuse_pass.cc | 20 ++++++++----------- .../softplus_activation_mkldnn_fuse_pass.h | 6 ++---- paddle/fluid/operators/activation_op.cc | 20 +++++++++++++++++++ 9 files changed, 53 insertions(+), 57 deletions(-) diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc index 913bb84255ed57..bd07967757b8a9 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.cc @@ -22,26 +22,21 @@ namespace paddle { namespace framework { namespace ir { -using paddle::platform::GetAttributeMap; -using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void ConvActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - auto act_types = GetSupportedActivations(); + auto act_types = paddle::platform::GetSupportedActivations(); std::vector conv_types = {"conv2d"}; for (const auto& conv_type : conv_types) for (auto& act_type : act_types) { - auto attr_map = GetAttributeMap(act_type); - FuseConvAct(graph, conv_type, act_type, attr_map); + FuseConvAct(graph, conv_type, act_type); } } -void ConvActivationMkldnnFusePass::FuseConvAct( - Graph* graph, - const std::string& conv_type, - std::string& act_type, - const std::unordered_map& attr_map) const { +void ConvActivationMkldnnFusePass::FuseConvAct(Graph* graph, + const std::string& conv_type, + std::string& act_type) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); FusePassBase::Init(conv_type + "_" + act_type + "_mkldnn_fuse_pass", graph); @@ -69,6 +64,7 @@ void ConvActivationMkldnnFusePass::FuseConvAct( OpDesc* conv_op = conv->Op(); OpDesc* act_op = activation->Op(); + auto attr_map = paddle::platform::GetAttributeMap(act_type); for (const auto& attrs : attr_map) { if (act_op->HasAttr(attrs.first)) { conv_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); diff --git a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h index e1e2898384609e..11925e1992df4c 100644 --- a/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/conv_activation_mkldnn_fuse_pass.h @@ -31,11 +31,9 @@ class ConvActivationMkldnnFusePass : public FusePassBase { protected: void ApplyImpl(Graph *graph) const override; - void FuseConvAct( - Graph *graph, - const std::string &conv_type, - std::string &act_type, - const std::unordered_map &attrs_map) const; + void FuseConvAct(Graph *graph, + const std::string &conv_type, + std::string &act_type) const; }; } // namespace ir diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc index 85a1de4357abc8..c9eee31606cc36 100644 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc @@ -24,27 +24,23 @@ namespace paddle { namespace framework { namespace ir { -using paddle::platform::GetAttributeMap; -using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void ElementwiseActivationOneDNNPass::ApplyImpl(Graph *graph) const { - auto act_types = GetSupportedActivations(); + auto act_types = paddle::platform::GetSupportedActivations(); std::vector elt_types = { "elementwise_add", "elementwise_sub", "elementwise_mul"}; for (const auto &elt_type : elt_types) for (const auto &act_type : act_types) { - auto attr_map = GetAttributeMap(act_type); - FuseElementwiseAct(graph, elt_type, act_type, attr_map); + FuseElementwiseAct(graph, elt_type, act_type); } } void ElementwiseActivationOneDNNPass::FuseElementwiseAct( Graph *graph, const std::string &elt_type, - const std::string &act_type, - const std::unordered_map &attr_map) const { + const std::string &act_type) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); FusePassBase::Init(elt_type + "_" + act_type + "_mkldnn_fuse_pass", graph); @@ -80,6 +76,7 @@ void ElementwiseActivationOneDNNPass::FuseElementwiseAct( } auto *activation_op = activation->Op(); + auto attr_map = paddle::platform::GetAttributeMap(act_type); for (const auto &attr : attr_map) { if (activation_op->HasAttr(attr.first)) { elementwise_op->SetAttr(attr.second, diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h index 8df479e3ddf06d..37bd5345ec78f4 100644 --- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h @@ -34,11 +34,9 @@ class ElementwiseActivationOneDNNPass : public FusePassBase { protected: void ApplyImpl(Graph *graph) const override; - void FuseElementwiseAct( - Graph *graph, - const std::string &elt_types, - const std::string &act_types, - const std::unordered_map &attr_map) const; + void FuseElementwiseAct(Graph *graph, + const std::string &elt_types, + const std::string &act_types) const; }; } // namespace ir diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc index 1996c11710ccf7..80f49c97e84658 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.cc @@ -22,26 +22,20 @@ namespace paddle { namespace framework { namespace ir { -using paddle::platform::GetAttributeMap; -using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void MatmulActivationMkldnnFusePass::ApplyImpl(Graph* graph) const { - auto act_types = GetSupportedActivations(); + auto act_types = paddle::platform::GetSupportedActivations(); std::vector matmul_types = {"matmul"}; for (const auto& matmul_type : matmul_types) for (auto& act_type : act_types) { - auto attr_map = GetAttributeMap(act_type); - FuseMatmulAct(graph, matmul_type, act_type, attr_map); + FuseMatmulAct(graph, matmul_type, act_type); } } void MatmulActivationMkldnnFusePass::FuseMatmulAct( - Graph* graph, - const std::string& matmul_type, - std::string& act_type, - const std::unordered_map& attr_map) const { + Graph* graph, const std::string& matmul_type, std::string& act_type) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); FusePassBase::Init(matmul_type + "_" + act_type + "_mkldnn_fuse_pass", graph); @@ -70,6 +64,7 @@ void MatmulActivationMkldnnFusePass::FuseMatmulAct( OpDesc* matmul_op = matmul->Op(); OpDesc* act_op = activation->Op(); + auto attr_map = paddle::platform::GetAttributeMap(act_type); for (const auto& attrs : attr_map) { if (act_op->HasAttr(attrs.first)) { matmul_op->SetAttr(attrs.second, act_op->GetAttr(attrs.first)); diff --git a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h index 0e8d613e8ae9d1..ebef63e2924386 100644 --- a/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/matmul_activation_mkldnn_fuse_pass.h @@ -31,11 +31,9 @@ class MatmulActivationMkldnnFusePass : public FusePassBase { protected: void ApplyImpl(Graph *graph) const override; - void FuseMatmulAct( - Graph *graph, - const std::string &matmul_type, - std::string &act_type, - const std::unordered_map &attrs_map) const; + void FuseMatmulAct(Graph *graph, + const std::string &matmul_type, + std::string &act_type) const; }; } // namespace ir diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc index 729b2327684107..41e70e529bf73d 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc @@ -24,23 +24,18 @@ namespace paddle { namespace framework { namespace ir { -using paddle::platform::GetAttributeMap; -using paddle::platform::GetSupportedActivations; using string::PrettyLogDetail; void SoftplusActivationOneDNNPass::ApplyImpl(Graph *graph) const { - auto act_types = GetSupportedActivations(); + auto act_types = paddle::platform::GetSupportedActivations(); for (const auto &act_type : act_types) { - auto attr_map = GetAttributeMap(act_type); - FuseSoftplusActivation(graph, act_type, attr_map); + FuseSoftplusActivation(graph, act_type); } } void SoftplusActivationOneDNNPass::FuseSoftplusActivation( - Graph *graph, - const std::string &fuse_activation, - const std::unordered_map &attr_map) const { + Graph *graph, const std::string &act_type) const { PADDLE_ENFORCE_NOT_NULL( graph, platform::errors::InvalidArgument("Graph cannot be nullptr.")); FusePassBase::Init("softplus_activation", graph); @@ -48,7 +43,7 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( GraphPatternDetector gpd; patterns::OperatorActivation softplus_activation_pattern( gpd.mutable_pattern(), "softplus_activation"); - softplus_activation_pattern("softplus", fuse_activation); + softplus_activation_pattern("softplus", act_type); int found_softplus_activation_count = 0; auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph, @@ -75,17 +70,18 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( } auto *activation_op = activation->Op(); + auto attr_map = paddle::platform::GetAttributeMap(act_type); for (const auto &attr : attr_map) { if (activation_op->HasAttr(attr.first)) { softplus_op->SetAttr(attr.second, activation_op->GetAttr(attr.first)); } } - if (fuse_activation == "gelu" && activation_op->HasAttr("approximate") && + if (act_type == "gelu" && activation_op->HasAttr("approximate") && BOOST_GET_CONST(bool, activation_op->GetAttr("approximate"))) softplus_op->SetAttr("fuse_activation", std::string("gelu_tanh")); else - softplus_op->SetAttr("fuse_activation", fuse_activation); + softplus_op->SetAttr("fuse_activation", act_type); softplus_op->SetAttr("use_mkldnn", true); @@ -101,7 +97,7 @@ void SoftplusActivationOneDNNPass::FuseSoftplusActivation( if (!Has("disable_logs") || !Get("disable_logs")) PrettyLogDetail("--- fused %d softplus with %s activation", found_softplus_activation_count, - fuse_activation); + act_type); } } // namespace ir } // namespace framework diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h index f36f640d3833d3..6368a102b0e852 100644 --- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.h @@ -34,10 +34,8 @@ class SoftplusActivationOneDNNPass : public FusePassBase { protected: void ApplyImpl(ir::Graph *graph) const override; - void FuseSoftplusActivation( - ir::Graph *graph, - const std::string &fuse_activation, - const std::unordered_map &attr_map) const; + void FuseSoftplusActivation(ir::Graph *graph, + const std::string &act_type) const; }; } // namespace ir diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index e3eafa87a1a73c..0da38de5ead55d 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -492,6 +492,26 @@ class SoftplusOpMaker : public framework::OpProtoAndCheckerMaker { "(bool, default false) Only used in cudnn kernel, need install cudnn.") .SetDefault(false) .AsExtra(); + AddAttr( + "fuse_activation_type", + "Fused activation type used in softplus OneDNN kernel.") + .SetDefault("") + .AsExtra(); + AddAttr( + "fuse_activation_alpha", + "Fused activation alpha parameter type used in softplus OneDNN kernel.") + .SetDefault(0.0f) + .AsExtra(); + AddAttr( + "fuse_activation_beta", + "Fused activation beta parameter type used in softplus OneDNN kernel.") + .SetDefault(0.0f) + .AsExtra(); + AddAttr( + "fuse_activation_scale", + "Fused activation scale parameter type used in softplus OneDNN kernel.") + .SetDefault(1.0f) + .AsExtra(); AddComment(R"DOC( :strong:`Softplus Activation Operator` From ba17cd4006a80fd4ae88220481c624688af70b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Tue, 5 Jul 2022 04:18:19 -0700 Subject: [PATCH 25/27] trigger CI From 991431dda78862597682e2c0414b6a9f6421b358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Wed, 6 Jul 2022 01:15:04 -0700 Subject: [PATCH 26/27] validate attribute --- paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc | 6 +----- paddle/fluid/platform/mkldnn_reuse.h | 7 ++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc index f676dca7d64ae7..f61d14489be2c3 100644 --- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc @@ -583,11 +583,7 @@ class ConvMKLDNNHandlerT post_operations.append_sum(sum_scale); } - bool has_activation = !ctx.Attr("fuse_activation").empty(); - if (has_activation) { - paddle::platform::AppendActivation( - ctx, post_operations, activation_scale); - } + paddle::platform::AppendActivation(ctx, post_operations, activation_scale); conv_attr.set_post_ops(post_operations); return conv_attr; diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h index cd3a28382f5d6f..e4d256a38ac3c7 100644 --- a/paddle/fluid/platform/mkldnn_reuse.h +++ b/paddle/fluid/platform/mkldnn_reuse.h @@ -1011,7 +1011,12 @@ class ActivationMKLDNNHandler static void AppendActivation(const framework::ExecutionContext& ctx, dnnl::post_ops& post_ops, float activation_scale = 1.0f) { - if (!ctx.HasAttr("fuse_activation")) return; + const auto invalid_attribute = + ctx.HasAttr("fuse_activation") + ? ctx.Attr("fuse_activation").empty() + : true; + if (invalid_attribute) return; + const auto fuse_activation = ctx.Attr("fuse_activation"); const auto fuse_alpha = ctx.HasAttr("fuse_alpha") ? ctx.Attr("fuse_alpha") : 0.0f; From 7e176a123882a6adb46a4a9d424f80f0d6027aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Mon, 11 Jul 2022 01:54:28 -0700 Subject: [PATCH 27/27] trim unit test --- python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt | 2 -- .../ir/inference/test_mkldnn_matmul_activation_fuse_pass.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt index da582fa3db0339..6f79a248cf38b6 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt @@ -216,7 +216,5 @@ if(WITH_GPU AND TENSORRT_FOUND) PROPERTIES TIMEOUT 120) set_tests_properties(test_mkldnn_conv_affine_channel_fuse_pass PROPERTIES TIMEOUT 60) - set_tests_properties(test_mkldnn_matmul_activation_fuse_pass - PROPERTIES TIMEOUT 300) endif() endif() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index 06488e5e769040..20028fb335b8fc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -119,7 +119,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis(quant=False, - max_examples=100, + max_examples=30, passes=['matmul_activation_mkldnn_fuse_pass'])