From 2b68c0e19c221874a28f03cc1ae46a5c5dbc1550 Mon Sep 17 00:00:00 2001 From: stalary Date: Fri, 9 Jul 2021 16:08:42 +0800 Subject: [PATCH 01/14] ADD: support sql block rule --- fe/fe-core/SchemaChangeV2Test | Bin 0 -> 800 bytes fe/fe-core/src/main/cup/sql_parser.cup | 48 +++- .../doris/analysis/AlterSqlBlockRuleStmt.java | 84 +++++++ .../analysis/CreateSqlBlockRuleStmt.java | 122 ++++++++++ .../doris/analysis/DropSqlBlockRuleStmt.java | 33 +++ .../doris/analysis/ShowSqlBlockRuleStmt.java | 55 +++++ .../apache/doris/analysis/UserIdentity.java | 4 + .../org/apache/doris/block/SqlBlockRule.java | 111 +++++++++ .../apache/doris/block/SqlBlockRuleMgr.java | 228 ++++++++++++++++++ .../org/apache/doris/catalog/Catalog.java | 22 +- .../java/org/apache/doris/common/Config.java | 11 +- .../apache/doris/common/FeMetaVersion.java | 4 +- .../org/apache/doris/common/MetaReader.java | 1 + .../org/apache/doris/common/MetaWriter.java | 2 + .../org/apache/doris/metric/MetricRepo.java | 4 + .../doris/mysql/privilege/PaloAuth.java | 2 +- .../org/apache/doris/persist/EditLog.java | 28 +++ .../apache/doris/persist/OperationType.java | 5 + .../java/org/apache/doris/qe/DdlExecutor.java | 9 + .../org/apache/doris/qe/ShowExecutor.java | 14 +- .../org/apache/doris/qe/StmtExecutor.java | 37 +++ fe/fe-core/src/main/jflex/sql_scanner.flex | 1 + .../org/apache/doris/qe/StmtExecutorTest.java | 15 ++ 23 files changed, 832 insertions(+), 8 deletions(-) create mode 100644 fe/fe-core/SchemaChangeV2Test create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java diff --git a/fe/fe-core/SchemaChangeV2Test b/fe/fe-core/SchemaChangeV2Test new file mode 100644 index 0000000000000000000000000000000000000000..0a76f63e4bc900c3b49405d1bbb2496feeda8a65 GIT binary patch literal 800 zcmcgq%WlFj5Jc*YZ|DK}+*S@!De4U}LA$Y)3*v2d=L5HkH zg^{-ne5Ud-Q;=GcRpYZ1n|IhLu{y_e|0U)ph4QSb3bXY zSj<@kX7(2Bg1e+Ud;wBndLy2ie9mU@D5a+yF_#}lMaZKrDg<*>A=0NdyTd6P`=clq zQe%*3%~3JV+#XSwu~LCeGs$MP*&BVZ?!-D=;cb}4kL0+=Ee}!@+8ucy6gECiM4PJSVbD$4S5#KgUFx0I9}SOX>(*PnXQ}P zi3X*e2BZwt%6PeYDFMCXqJq%ekvHfMAH9J$q)*fhoE5yVa~&zoU>a+&)>bvZolx36 F`~e+&-d+Fz literal 0 HcmV?d00001 diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index c360419d0de41c..5b67d85d966217 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -270,7 +270,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_A KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNSIGNED, KW_USE, KW_USER, KW_USING, KW_UNINSTALL, KW_VALUE, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VERBOSE, KW_VIEW, KW_WARNINGS, KW_WEEK, KW_WHEN, KW_WHITELIST, KW_WHERE, KW_WITH, KW_WORK, KW_WRITE, - KW_YEAR; + KW_YEAR, KW_SQL_BLOCK_RULE; terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT; terminal BITAND, BITOR, BITXOR, BITNOT; @@ -295,7 +295,8 @@ nonterminal StatementBase stmt, show_stmt, show_param, help_stmt, load_stmt, use_stmt, kill_stmt, drop_stmt, recover_stmt, grant_stmt, revoke_stmt, create_stmt, set_stmt, sync_stmt, cancel_stmt, cancel_param, delete_stmt, link_stmt, migrate_stmt, enter_stmt, transaction_stmt, unsupported_stmt, export_stmt, admin_stmt, truncate_stmt, import_columns_stmt, import_delete_on_stmt, import_sequence_stmt, import_where_stmt, install_plugin_stmt, uninstall_plugin_stmt, - import_preceding_filter_stmt; + import_preceding_filter_stmt, + create_sql_block_rule_stmt, alter_sql_block_rule_stmt, drop_sql_block_rule_stmt, show_sql_block_rule_stmt; nonterminal String transaction_label; nonterminal ImportColumnDesc import_column_desc; @@ -723,6 +724,14 @@ stmt ::= {: RESULT = stmt; :} | uninstall_plugin_stmt : stmt {: RESULT = stmt; :} + | create_sql_block_rule_stmt : stmt + {: RESULT = stmt; :} + | alter_sql_block_rule_stmt : stmt + {: RESULT = stmt; :} + | drop_sql_block_rule_stmt : stmt + {: RESULT = stmt; :} + | show_sql_block_rule_stmt : stmt + {: RESULT = stmt; :} | /* empty: query only has comments */ {: RESULT = new EmptyStmt(); @@ -1818,6 +1827,41 @@ show_create_routine_load_stmt ::= :} ; +// Sql block rule statement +create_sql_block_rule_stmt ::= + KW_CREATE KW_SQL_BLOCK_RULE STRING_LITERAL:ruleName + opt_properties:properties + {: + RESULT = new CreateSqlBlockRuleStmt(ruleName, properties); + :} + ; + +alter_sql_block_rule_stmt ::= + KW_ALTER KW_SQL_BLOCK_RULE STRING_LITERAL:ruleName + opt_properties:properties + {: + RESULT = new AlterSqlBlockRuleStmt(ruleName, properties); + :} + ; + +drop_sql_block_rule_stmt ::= + KW_DROP KW_SQL_BLOCK_RULE ident_list:ruleNames + {: + RESULT = new DropSqlBlockRuleStmt(ruleNames); + :} + ; + +show_sql_block_rule_stmt ::= + KW_SHOW KW_SQL_BLOCK_RULE KW_FOR ident:ruleName + {: + RESULT = new ShowSqlBlockRuleStmt(ruleName); + :} + | KW_SHOW KW_SQL_BLOCK_RULE + {: + RESULT = new ShowSqlBlockRuleStmt(null); + :} + ; + // Grant statement grant_stmt ::= KW_GRANT privilege_list:privs KW_ON tbl_pattern:tblPattern KW_TO user_identity:userId diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java new file mode 100644 index 00000000000000..9dba978565c032 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.block.SqlBlockRule; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; + +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class AlterSqlBlockRuleStmt extends DdlStmt { + + private final String ruleName; + + private String user; + + private String sql; + + private Boolean enable; + + private final Map properties; + + public AlterSqlBlockRuleStmt(String ruleName, Map properties) { + this.ruleName = ruleName; + this.properties = properties; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + // check properties + checkProperties(properties); + } + + private void checkProperties(Map properties) throws UserException { + this.user = properties.get(CreateSqlBlockRuleStmt.USER_PROPERTY); + // if not default, need check whether user exist + if (StringUtils.isNotEmpty(user) && !SqlBlockRule.DEFAULT_USER.equals(user)) { + List allUserIdents = Catalog.getCurrentCatalog().getAuth().getAllUserIdents(false).stream().map(UserIdentity::getUser).collect(Collectors.toList()); + if (!allUserIdents.contains(user)) { + throw new AnalysisException(user + " is not exist"); + } + } + this.sql = properties.get(CreateSqlBlockRuleStmt.SQL_PROPERTY); + String enableStr = properties.get(CreateSqlBlockRuleStmt.ENABLE_PROPERTY); + this.enable = StringUtils.isNotEmpty(enableStr) ? Boolean.parseBoolean(enableStr) : null; + } + + public String getRuleName() { + return ruleName; + } + + public String getUser() { + return user; + } + + public String getSql() { + return sql; + } + + public Boolean getEnable() { + return enable; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java new file mode 100644 index 00000000000000..251318b23502ff --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.block.SqlBlockRule; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/* + Create sqlBlockRule statement + + syntax: + CREATE SQL_BLOCK_RULE LOAD NAME PROPERTIES + ( + user = default, + sql = select * from a, + enable = true + ) +*/ +public class CreateSqlBlockRuleStmt extends DdlStmt { + + public static final String NAME_PROPERTY = "name"; + + public static final String USER_PROPERTY = "user"; + + public static final String SQL_PROPERTY = "sql"; + + public static final String ENABLE_PROPERTY = "enable"; + + private final String ruleName; + + // default stands for all users + private String user; + + private String sql; + + // whether to use the rule + private boolean enable; + + private final Map properties; + + private static final String NAME_TYPE = "SQL BLOCK RULE NAME"; + + private static final ImmutableSet PROPERTIES_SET = new ImmutableSet.Builder() + .add(NAME_PROPERTY) + .add(USER_PROPERTY) + .add(SQL_PROPERTY) + .add(ENABLE_PROPERTY) + .build(); + + public CreateSqlBlockRuleStmt(String ruleName, Map properties) { + this.ruleName = ruleName; + this.properties = properties; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + // check name + FeNameFormat.checkCommonName(NAME_TYPE, ruleName); + // check properties + checkProperties(properties); + } + + private void checkProperties(Map properties) throws UserException { + Optional optional = properties.keySet().stream().filter( + entity -> !PROPERTIES_SET.contains(entity)).findFirst(); + if (optional.isPresent()) { + throw new AnalysisException(optional.get() + " is invalid property"); + } + this.user = properties.get(USER_PROPERTY); + // if not default, need check whether user exist + if (!SqlBlockRule.DEFAULT_USER.equals(user)) { + List allUserIdents = Catalog.getCurrentCatalog().getAuth().getAllUserIdents(false).stream().map(UserIdentity::getUser).collect(Collectors.toList()); + if (!allUserIdents.contains(user)) { + throw new AnalysisException(user + " is not exist"); + } + } + this.sql = properties.get(SQL_PROPERTY); + this.enable = Util.getBooleanPropertyOrDefault(properties.get(ENABLE_PROPERTY), true, ENABLE_PROPERTY + " should be a boolean"); + } + + public String getRuleName() { + return ruleName; + } + + public String getUser() { + return user; + } + + public String getSql() { + return sql; + } + + public boolean isEnable() { + return enable; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java new file mode 100644 index 00000000000000..946b76e9145887 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import java.util.List; + +public class DropSqlBlockRuleStmt extends DdlStmt { + + private List ruleNames; + + public DropSqlBlockRuleStmt(List ruleNames) { + this.ruleNames = ruleNames; + } + + public List getRuleNames() { + return ruleNames; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java new file mode 100644 index 00000000000000..edb3500d075656 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.qe.ShowResultSetMetaData; + +/* + Create sqlBlockRule statement + + syntax: + show sql_block_rule + show sql_block_rule for rule_name +*/ +public class ShowSqlBlockRuleStmt extends ShowStmt { + + private static final ShowResultSetMetaData META_DATA = + ShowResultSetMetaData.builder() + .addColumn(new Column("Name", ScalarType.createVarchar(50))) + .addColumn(new Column("User", ScalarType.createVarchar(30))) + .addColumn(new Column("Sql", ScalarType.createVarchar(65535))) + .addColumn(new Column("Enable", ScalarType.createVarchar(4))) + .build(); + + private String ruleName; // optional + + public ShowSqlBlockRuleStmt(String ruleName) { + this.ruleName = ruleName; + } + + public String getRuleName() { + return ruleName; + } + + @Override + public ShowResultSetMetaData getMetaData() { + return META_DATA; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java index 51822d1c75cfe6..03a457eb086b22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java @@ -97,6 +97,10 @@ public String getQualifiedUser() { return user; } + public String getUser() { + return user; + } + public String getHost() { return host; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java b/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java new file mode 100644 index 00000000000000..d2449f3dc99e72 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.block; + +import org.apache.doris.analysis.AlterSqlBlockRuleStmt; +import org.apache.doris.analysis.CreateSqlBlockRuleStmt; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; + +import com.google.common.collect.Lists; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +public class SqlBlockRule implements Writable { + + public static final String NAME_TYPE = "SQL BLOCK RULE NAME"; + + public static final String DEFAULT_USER = "default"; + + // the rule name, cluster unique + private String name; + + // default stands for all users + private String user; + + private String sql; + + // whether to use the rule + private Boolean enable; + + public SqlBlockRule(String name) { + this.name = name; + } + + public SqlBlockRule(String name, String user, String sql, Boolean enable) { + this.name = name; + this.user = user; + this.sql = sql; + this.enable = enable; + } + + public static SqlBlockRule fromCreateStmt(CreateSqlBlockRuleStmt stmt) { + return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.isEnable()); + } + + public static SqlBlockRule fromAlterStmt(AlterSqlBlockRuleStmt stmt) { + return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.getEnable()); + } + + public String getName() { + return name; + } + + public String getUser() { + return user; + } + + public String getSql() { + return sql; + } + + public Boolean getEnable() { + return enable; + } + + public void setUser(String user) { + this.user = user; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public void setEnable(Boolean enable) { + this.enable = enable; + } + + public List getShowInfo() { + return Lists.newArrayList(this.name, this.user, this.sql, String.valueOf(this.enable)); + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, name); + Text.writeString(out, user); + Text.writeString(out, sql); + Text.writeString(out, String.valueOf(enable)); + } + + public static SqlBlockRule read(DataInput in) throws IOException { + return new SqlBlockRule(Text.readString(in), Text.readString(in), Text.readString(in), Boolean.valueOf(Text.readString(in))); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java new file mode 100644 index 00000000000000..a08e4e9341cd84 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.block; + +import org.apache.doris.analysis.AlterSqlBlockRuleStmt; +import org.apache.doris.analysis.CreateSqlBlockRuleStmt; +import org.apache.doris.analysis.DropSqlBlockRuleStmt; +import org.apache.doris.analysis.ShowSqlBlockRuleStmt; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Writable; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class SqlBlockRuleMgr implements Writable { + private static final Logger LOG = LogManager.getLogger(SqlBlockRuleMgr.class); + + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + + private Map> userToSqlBlockRuleMap = Maps.newConcurrentMap(); + + private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); + + private void writeLock() { + lock.writeLock().lock(); + } + + private void writeUnlock() { + lock.writeLock().unlock(); + } + + public boolean existRule(String name) { + return nameToSqlBlockRuleMap.containsKey(name); + } + + public List get(ShowSqlBlockRuleStmt stmt) throws AnalysisException { + String ruleName = stmt.getRuleName(); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + if (StringUtils.isNotEmpty(ruleName)) { + if (nameToSqlBlockRuleMap.containsKey(ruleName)) { + SqlBlockRule sqlBlockRule = nameToSqlBlockRuleMap.get(ruleName); + return Lists.newArrayList(sqlBlockRule); + } + return Lists.newArrayList(); + } + return Lists.newArrayList(nameToSqlBlockRuleMap.values()); + } + + public void createSqlBlockRule(CreateSqlBlockRuleStmt stmt) throws UserException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + SqlBlockRule sqlBlockRule = SqlBlockRule.fromCreateStmt(stmt); + create(sqlBlockRule); + } + + private void create(SqlBlockRule sqlBlockRule) throws DdlException { + writeLock(); + try { + String ruleName = sqlBlockRule.getName(); + if (existRule(ruleName)) { + throw new DdlException("the sql block rule " + ruleName + " already create"); + } + Catalog.getCurrentCatalog().getEditLog().logCreateSqlBlockRule(sqlBlockRule); + unprotectedAdd(sqlBlockRule); + } finally { + writeUnlock(); + } + } + + public void replayCreate(SqlBlockRule sqlBlockRule) { + unprotectedAdd(sqlBlockRule); + LOG.info("replay create sql block rule: {}", sqlBlockRule); + } + + public void alterSqlBlockRule(AlterSqlBlockRuleStmt stmt) throws UserException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + SqlBlockRule sqlBlockRule = SqlBlockRule.fromAlterStmt(stmt); + alter(sqlBlockRule); + } + + private void alter(SqlBlockRule sqlBlockRule) throws DdlException { + writeLock(); + try { + String ruleName = sqlBlockRule.getName(); + if (!existRule(ruleName)) { + throw new DdlException("the sql block rule " + ruleName + " not exist"); + } + SqlBlockRule originRule = nameToSqlBlockRuleMap.get(ruleName); + if (StringUtils.isEmpty(sqlBlockRule.getUser())) { + sqlBlockRule.setUser(originRule.getUser()); + } + if (StringUtils.isEmpty(sqlBlockRule.getSql())) { + sqlBlockRule.setSql(originRule.getSql()); + } + if (sqlBlockRule.getEnable() == null) { + sqlBlockRule.setEnable(originRule.getEnable()); + } + Catalog.getCurrentCatalog().getEditLog().logAlterSqlBlockRule(sqlBlockRule); + unprotectedUpdate(sqlBlockRule); + } finally { + writeUnlock(); + } + } + + public void replayAlter(SqlBlockRule sqlBlockRule) { + unprotectedUpdate(sqlBlockRule); + LOG.info("replay alter sql block rule: {}", sqlBlockRule); + } + + public void unprotectedUpdate(SqlBlockRule sqlBlockRule) { + nameToSqlBlockRuleMap.put(sqlBlockRule.getName(), sqlBlockRule); + List sqlBlockRules = userToSqlBlockRuleMap.getOrDefault(sqlBlockRule.getUser(), new ArrayList<>()); + sqlBlockRules.removeIf(rule -> sqlBlockRule.getName().equals(rule.getName())); + sqlBlockRules.add(sqlBlockRule); + userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); + } + + public void unprotectedAdd(SqlBlockRule sqlBlockRule) { + nameToSqlBlockRuleMap.put(sqlBlockRule.getName(), sqlBlockRule); + List sqlBlockRules = userToSqlBlockRuleMap.getOrDefault(sqlBlockRule.getUser(), new ArrayList<>()); + sqlBlockRules.add(sqlBlockRule); + userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); + } + + public void dropSqlBlockRule(DropSqlBlockRuleStmt stmt) throws UserException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + List ruleNames = stmt.getRuleNames(); + drop(ruleNames); + } + + public void drop(List ruleNames) throws DdlException { + writeLock(); + try { + for (String ruleName : ruleNames) { + if (!existRule(ruleName)) { + throw new DdlException("the sql block rule " + ruleName + " not exist"); + } + SqlBlockRule sqlBlockRule = nameToSqlBlockRuleMap.get(ruleName); + if (sqlBlockRule == null) { + continue; + } + Catalog.getCurrentCatalog().getEditLog().logDropSqlBlockRule(sqlBlockRule); + unprotectedDrop(sqlBlockRule); + } + } finally { + writeUnlock(); + } + } + + public void replayDrop(SqlBlockRule sqlBlockRule) { + unprotectedDrop(sqlBlockRule); + LOG.info("replay drop sql block rule: {}", sqlBlockRule); + } + + public void unprotectedDrop(SqlBlockRule sqlBlockRule) { + nameToSqlBlockRuleMap.remove(sqlBlockRule.getName()); + List sqlBlockRules = userToSqlBlockRuleMap.get(sqlBlockRule.getUser()); + sqlBlockRules.removeIf(rule -> sqlBlockRule.getName().equals(rule.getName())); + userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); + } + + public Map> getUserToSqlBlockRuleMap() { + return userToSqlBlockRuleMap; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(nameToSqlBlockRuleMap.size()); + for (SqlBlockRule sqlBlockRule : nameToSqlBlockRuleMap.values()) { + sqlBlockRule.write(out); + } + } + + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + SqlBlockRule read = SqlBlockRule.read(in); + unprotectedAdd(read); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index fdc91477bd46e0..67e052a5aef281 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -85,6 +85,7 @@ import org.apache.doris.analysis.UserDesc; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupHandler; +import org.apache.doris.block.SqlBlockRuleMgr; import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.Database.DbState; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; @@ -308,6 +309,7 @@ public class Catalog { private LoadManager loadManager; private StreamLoadRecordMgr streamLoadRecordMgr; private RoutineLoadManager routineLoadManager; + private SqlBlockRuleMgr sqlBlockRuleMgr; private ExportMgr exportMgr; private SyncJobManager syncJobManager; private Alter alter; @@ -495,6 +497,7 @@ private Catalog(boolean isCheckpointCatalog) { this.fullNameToDb = new ConcurrentHashMap<>(); this.load = new Load(); this.routineLoadManager = new RoutineLoadManager(); + this.sqlBlockRuleMgr = new SqlBlockRuleMgr(); this.exportMgr = new ExportMgr(); this.syncJobManager = new SyncJobManager(); this.alter = new Alter(); @@ -1901,6 +1904,14 @@ public long loadSmallFiles(DataInputStream in, long checksum) throws IOException return checksum; } + public long loadSqlBlockRule(DataInputStream in, long checksum) throws IOException { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_100) { + sqlBlockRuleMgr.readFields(in); + } + LOG.info("finished replay sqlBlockRule from image"); + return checksum; + } + // Only called by checkpoint thread public void saveImage() throws IOException { // Write image.ckpt @@ -1924,7 +1935,7 @@ public void saveImage(File curFile, long replayedJournalId) throws IOException { MetaWriter.write(curFile, this); } - public long saveHeader(CountingDataOutputStream dos, long replayedJournalId, long checksum) throws IOException { + public long saveHeader(DataOutputStream dos, long replayedJournalId, long checksum) throws IOException { // Write meta version checksum ^= FeConstants.meta_version; dos.writeInt(FeConstants.meta_version); @@ -2174,6 +2185,11 @@ public long saveSmallFiles(CountingDataOutputStream dos, long checksum) throws I return checksum; } + public long saveSqlBlockRule(DataOutputStream out, long checksum) throws IOException { + Catalog.getCurrentCatalog().getSqlBlocklistMgr().write(out); + return checksum; + } + public void createLabelCleaner() { labelCleaner = new MasterDaemon("LoadLabelCleaner", Config.label_clean_interval_second * 1000L) { @Override @@ -4864,6 +4880,10 @@ public RoutineLoadManager getRoutineLoadManager() { return routineLoadManager; } + public SqlBlockRuleMgr getSqlBlocklistMgr() { + return sqlBlockRuleMgr; + } + public RoutineLoadTaskScheduler getRoutineLoadTaskScheduler(){ return routineLoadTaskScheduler; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 95e89f85786f88..76a8d9a0946b9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1469,12 +1469,21 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static int default_max_query_instances = -1; - /* + /** * One master daemon thread will update global partition in memory info every partition_in_memory_update_interval_secs */ @ConfField(mutable = false, masterOnly = true) public static int partition_in_memory_update_interval_secs = 300; +<<<<<<< HEAD @ConfField(masterOnly = true) public static boolean enable_concurrent_update = false; +======= + + /** + * Whether to enabel block sql + */ + @ConfField(mutable = true, masterOnly = false) + public static boolean enable_sql_block = false; +>>>>>>> 2f8cca53c (ADD: support sql block rule) } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java index 149f3e19af2d66..0aec044b4068bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -218,6 +218,8 @@ public final class FeMetaVersion { public static final int VERSION_102 = 102; // support sync job public static final int VERSION_103 = 103; + // add sql block rule to deny specified sql + public static final int VERSION_104 = 104; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_103; + public static final int VERSION_CURRENT = VERSION_104; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/MetaReader.java b/fe/fe-core/src/main/java/org/apache/doris/common/MetaReader.java index e0e30c1ce7574f..be646f96ff6f81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/MetaReader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/MetaReader.java @@ -102,6 +102,7 @@ public static void read(File imageFile, Catalog catalog) throws IOException, Ddl checksum = catalog.loadSmallFiles(dis, checksum); checksum = catalog.loadPlugins(dis, checksum); checksum = catalog.loadDeleteHandler(dis, checksum); + checksum = catalog.loadSqlBlockRule(dis, checksum); } MetaFooter metaFooter = MetaFooter.read(imageFile); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/MetaWriter.java b/fe/fe-core/src/main/java/org/apache/doris/common/MetaWriter.java index b591d0855009d1..a58f4f5443f905 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/MetaWriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/MetaWriter.java @@ -122,6 +122,8 @@ public static void write(File imageFile, Catalog catalog) throws IOException { checksum = catalog.savePlugins(dos, checksum); metaIndices.add(new MetaIndex("deleteHandler", dos.getCount())); checksum = catalog.saveDeleteHandler(dos, checksum); + metaIndices.add(new MetaIndex("sqlBlockRule", dos.getCount())); + checksum = catalog.saveSqlBlockRule(dos, checksum); } MetaFooter.write(imageFile, metaIndices, checksum); diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 50a50d08f1c777..941857a738d6b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -92,6 +92,7 @@ public final class MetricRepo { public static LongCounterMetric COUNTER_ROUTINE_LOAD_ROWS; public static LongCounterMetric COUNTER_ROUTINE_LOAD_RECEIVED_BYTES; public static LongCounterMetric COUNTER_ROUTINE_LOAD_ERROR_ROWS; + public static LongCounterMetric COUNTER_HIT_SQL_BLOCK_RULE; public static Histogram HISTO_QUERY_LATENCY; public static Histogram HISTO_EDIT_LOG_WRITE_LATENCY; @@ -302,6 +303,9 @@ public Long getValue() { "total error rows of routine load"); PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_ROUTINE_LOAD_ERROR_ROWS); + COUNTER_HIT_SQL_BLOCK_RULE = new LongCounterMetric("counter_hit_sql_blacklist", MetricUnit.ROWS, + "total hit sql blacklist query"); + PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_HIT_SQL_BLOCK_RULE); // 3. histogram HISTO_QUERY_LATENCY = METRIC_REGISTER.histogram(MetricRegistry.name("query", "latency", "ms")); HISTO_EDIT_LOG_WRITE_LATENCY = METRIC_REGISTER.histogram(MetricRegistry.name("editlog", "write", "latency", "ms")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java index 209ed36742ed2f..8c9d7499cf018e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java @@ -1333,7 +1333,7 @@ private void getUserAuthInfo(List> userAuthInfos, UserIdentity user userAuthInfos.add(userAuthInfo); } - private Set getAllUserIdents(boolean includeEntrySetByResolver) { + public Set getAllUserIdents(boolean includeEntrySetByResolver) { Set userIdents = Sets.newHashSet(); for (PrivEntry entry : userPrivTable.entries) { if (!includeEntrySetByResolver && entry.isSetByDomainResolver()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index dd029f203afc2f..f2b7b3550354ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -26,6 +26,7 @@ import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; import org.apache.doris.backup.RestoreJob; +import org.apache.doris.block.SqlBlockRule; import org.apache.doris.catalog.BrokerMgr; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; @@ -843,6 +844,21 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { catalog.getAlterInstance().replayReplaceTable(log); break; } + case OperationType.OP_CREATE_SQL_BLOCK_RULE: { + SqlBlockRule rule = (SqlBlockRule) journal.getData(); + catalog.getSqlBlocklistMgr().replayCreate(rule); + break; + } + case OperationType.OP_ALTER_SQL_BLOCK_RULE: { + SqlBlockRule rule = (SqlBlockRule) journal.getData(); + catalog.getSqlBlocklistMgr().replayAlter(rule); + break; + } + case OperationType.OP_DROP_SQL_BLOCK_RULE: { + SqlBlockRule rule = (SqlBlockRule) journal.getData(); + catalog.getSqlBlocklistMgr().replayDrop(rule); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); @@ -1446,4 +1462,16 @@ public void logBatchRemoveTransactions(BatchRemoveTransactionsOperation op) { public void logModifyComment(ModifyCommentOperationLog op) { logEdit(OperationType.OP_MODIFY_COMMENT, op); } + + public void logCreateSqlBlockRule(SqlBlockRule rule) { + logEdit(OperationType.OP_CREATE_SQL_BLOCK_RULE, rule); + } + + public void logAlterSqlBlockRule(SqlBlockRule rule) { + logEdit(OperationType.OP_ALTER_SQL_BLOCK_RULE, rule); + } + + public void logDropSqlBlockRule(SqlBlockRule rule) { + logEdit(OperationType.OP_DROP_SQL_BLOCK_RULE, rule); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 6d194574989f5c..3916a76dc289b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -209,6 +209,11 @@ public class OperationType { public static final short OP_SET_LDAP_PASSWORD = 290; + // sql block rule 300-310 + public static final short OP_CREATE_SQL_BLOCK_RULE = 300; + public static final short OP_ALTER_SQL_BLOCK_RULE = 301; + public static final short OP_DROP_SQL_BLOCK_RULE = 302; + // get opcode name by op codeStri public static String getOpName(short opCode) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 1d2a5f59020800..c5c3e996a10f0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.AlterDatabaseQuotaStmt; import org.apache.doris.analysis.AlterDatabaseRename; import org.apache.doris.analysis.AlterRoutineLoadStmt; +import org.apache.doris.analysis.AlterSqlBlockRuleStmt; import org.apache.doris.analysis.AlterSystemStmt; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.AlterViewStmt; @@ -46,6 +47,7 @@ import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.analysis.CreateRoleStmt; import org.apache.doris.analysis.CreateRoutineLoadStmt; +import org.apache.doris.analysis.CreateSqlBlockRuleStmt; import org.apache.doris.analysis.CreateTableLikeStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.CreateUserStmt; @@ -61,6 +63,7 @@ import org.apache.doris.analysis.DropRepositoryStmt; import org.apache.doris.analysis.DropResourceStmt; import org.apache.doris.analysis.DropRoleStmt; +import org.apache.doris.analysis.DropSqlBlockRuleStmt; import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.DropUserStmt; import org.apache.doris.analysis.GrantStmt; @@ -263,6 +266,12 @@ public static void execute(Catalog catalog, DdlStmt ddlStmt) throws Exception { catalog.getSyncJobManager().stopSyncJob((StopSyncJobStmt) ddlStmt); } else if (ddlStmt instanceof AdminCleanTrashStmt) { catalog.cleanTrash((AdminCleanTrashStmt) ddlStmt); + } else if (ddlStmt instanceof CreateSqlBlockRuleStmt) { + catalog.getSqlBlocklistMgr().createSqlBlockRule((CreateSqlBlockRuleStmt) ddlStmt); + } else if (ddlStmt instanceof AlterSqlBlockRuleStmt) { + catalog.getSqlBlocklistMgr().alterSqlBlockRule((AlterSqlBlockRuleStmt) ddlStmt); + } else if (ddlStmt instanceof DropSqlBlockRuleStmt) { + catalog.getSqlBlocklistMgr().dropSqlBlockRule((DropSqlBlockRuleStmt) ddlStmt); } else { throw new DdlException("Unknown statement."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 83b00732445811..50dd526a79d542 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -68,6 +68,7 @@ import org.apache.doris.analysis.ShowRoutineLoadTaskStmt; import org.apache.doris.analysis.ShowSmallFilesStmt; import org.apache.doris.analysis.ShowSnapshotStmt; +import org.apache.doris.analysis.ShowSqlBlockRuleStmt; import org.apache.doris.analysis.ShowStmt; import org.apache.doris.analysis.ShowStreamLoadStmt; import org.apache.doris.analysis.ShowSyncJobStmt; @@ -85,6 +86,7 @@ import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; import org.apache.doris.backup.RestoreJob; +import org.apache.doris.block.SqlBlockRule; import org.apache.doris.catalog.BrokerMgr; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; @@ -313,6 +315,8 @@ public ShowResultSet execute() throws AnalysisException { handleAdminShowDataSkew(); } else if (stmt instanceof ShowSyncJobStmt) { handleShowSyncJobs(); + } else if (stmt instanceof ShowSqlBlockRuleStmt) { + handleShowSqlBlockRule(); } else { handleEmtpy(); } @@ -2087,7 +2091,13 @@ private void handleAdminShowDataSkew() throws AnalysisException { } resultSet = new ShowResultSet(showStmt.getMetaData(), results); } -} - + public void handleShowSqlBlockRule() throws AnalysisException { + ShowSqlBlockRuleStmt showStmt = (ShowSqlBlockRuleStmt) stmt; + List> rows = Lists.newArrayList(); + List sqlBlockRules = Catalog.getCurrentCatalog().getSqlBlocklistMgr().get(showStmt); + sqlBlockRules.forEach(rule -> rows.add(rule.getShowInfo())); + resultSet = new ShowResultSet(showStmt.getMetaData(), rows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 49bd19ded540ec..860de0147d50c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -45,6 +45,7 @@ import org.apache.doris.analysis.TransactionStmt; import org.apache.doris.analysis.UnsupportedStmt; import org.apache.doris.analysis.UseStmt; +import org.apache.doris.block.SqlBlockRule; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; @@ -108,6 +109,7 @@ import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -121,6 +123,7 @@ import java.io.IOException; import java.io.StringReader; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -128,6 +131,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; import java.util.stream.Collectors; // Do one COM_QUERY process. @@ -324,6 +328,11 @@ public void execute(TUniqueId queryId) throws Exception { } if (parsedStmt instanceof QueryStmt) { + boolean enableSqlBlock = Config.enable_sql_block; + if (enableSqlBlock) { + QueryStmt queryStmt = (QueryStmt) this.parsedStmt; + matchSql(queryStmt); + } context.getState().setIsQuery(true); MetricRepo.COUNTER_QUERY_BEGIN.increase(1L); int retryTime = Config.max_query_retry_time; @@ -1460,5 +1469,33 @@ public Data.PQueryStatistics getQueryStatisticsForAuditLog() { private List exprToType(List exprs) { return exprs.stream().map(e -> e.getType().getPrimitiveType()).collect(Collectors.toList()); } + + private void matchSql(QueryStmt queryStmt) throws AnalysisException { + Map> userToSqlBlockRuleMap = Catalog.getCurrentCatalog().getSqlBlocklistMgr().getUserToSqlBlockRuleMap(); + // match default rule + String sql = queryStmt.getOrigStmt().originStmt; + List defaultRules = userToSqlBlockRuleMap.getOrDefault(SqlBlockRule.DEFAULT_USER, new ArrayList<>()); + for (SqlBlockRule rule : defaultRules) { + matchSql(rule, sql); + } + // match user rule + String user = context.getUserIdentity().getUser(); + List userRules = userToSqlBlockRuleMap.getOrDefault(user, new ArrayList<>()); + for (SqlBlockRule rule : userRules) { + matchSql(rule, sql); + } + } + + @VisibleForTesting + public static void matchSql(SqlBlockRule rule, String sql) throws AnalysisException { + if (rule.getEnable() != null && rule.getEnable()) { + String sqlPattern = rule.getSql(); + Pattern pattern = Pattern.compile(sqlPattern); + if (pattern.matcher(sql).find()) { + MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); + throw new AnalysisException("query match sql block rule: " + rule.getName()); + } + } + } } diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 5e99627e3f77f3..3fa6fb6a378987 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -408,6 +408,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("write", new Integer(SqlParserSymbols.KW_WRITE)); keywordMap.put("year", new Integer(SqlParserSymbols.KW_YEAR)); keywordMap.put("||", new Integer(SqlParserSymbols.KW_PIPE)); + keywordMap.put("sql_block_rule", new Integer(SqlParserSymbols.KW_SQL_BLOCK_RULE)); } // map from token id to token description diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 506b08ca9da11e..93045f4101b291 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -30,7 +30,9 @@ import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UseStmt; +import org.apache.doris.block.SqlBlockRule; import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.RuntimeProfile; @@ -732,5 +734,18 @@ public void testUseFail(@Mocked UseStmt useStmt, @Mocked SqlParser parser) throw Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); } + + @Test(expected = AnalysisException.class) + public void testMatchSql() throws AnalysisException { +// String sql = "select * from test_table join limit 10"; +// SqlBlockRule rule = new SqlBlockRule("test_rule", "default", "select \\* from .*", true); +// StmtExecutor.matchSql(rule, sql); +// String sql1 = "select * from test_table limit 5"; +// SqlBlockRule rule1 = new SqlBlockRule("test_rule1", "default", "select \\* from test_table limit [1-9]", true); +// StmtExecutor.matchSql(rule1, sql1); + String sql2 = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; + SqlBlockRule rule2 = new SqlBlockRule("test_rule1", "default", ".* join .*", true); + StmtExecutor.matchSql(rule2, sql2); + } } From 56e76d6246b7a3827bc915e7b8b884923322bcbe Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 13 Jul 2021 22:50:57 +0800 Subject: [PATCH 02/14] ADD: mod some code, add test, support sql hash match --- docs/.vuepress/sidebar/en.js | 1 + docs/.vuepress/sidebar/zh-CN.js | 1 + .../block-rule/sql-block.md | 45 ++++++++++ .../block-rule/sql-block.md | 45 ++++++++++ fe/fe-core/src/main/cup/sql_parser.cup | 67 +++++---------- .../doris/analysis/AlterSqlBlockRuleStmt.java | 32 +++++-- .../analysis/CreateSqlBlockRuleStmt.java | 56 ++++++++---- .../doris/analysis/DropSqlBlockRuleStmt.java | 16 ++++ .../{block => blockrule}/SqlBlockRule.java | 29 +++++-- .../{block => blockrule}/SqlBlockRuleMgr.java | 85 +++++++++++-------- .../org/apache/doris/catalog/Catalog.java | 6 +- .../apache/doris/journal/JournalEntity.java | 16 ++++ .../org/apache/doris/metric/MetricRepo.java | 4 +- .../doris/mysql/privilege/PaloAuth.java | 2 +- .../org/apache/doris/persist/EditLog.java | 8 +- .../org/apache/doris/plugin/AuditEvent.java | 7 ++ .../org/apache/doris/qe/ConnectProcessor.java | 10 ++- .../java/org/apache/doris/qe/DdlExecutor.java | 6 +- .../org/apache/doris/qe/ShowExecutor.java | 5 +- .../org/apache/doris/qe/StmtExecutor.java | 36 ++------ .../analysis/AlterSqlBlockRuleStmtTest.java | 72 ++++++++++++++++ .../analysis/CreateSqlBlockRuleStmtTest.java | 75 ++++++++++++++++ .../doris/blockrule/SqlBlockRuleMgrTest.java | 78 +++++++++++++++++ .../org/apache/doris/qe/ShowExecutorTest.java | 13 +++ .../org/apache/doris/qe/StmtExecutorTest.java | 15 ---- 25 files changed, 549 insertions(+), 181 deletions(-) create mode 100644 docs/en/administrator-guide/block-rule/sql-block.md create mode 100644 docs/zh-CN/administrator-guide/block-rule/sql-block.md rename fe/fe-core/src/main/java/org/apache/doris/{block => blockrule}/SqlBlockRule.java (80%) rename fe/fe-core/src/main/java/org/apache/doris/{block => blockrule}/SqlBlockRuleMgr.java (77%) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/AlterSqlBlockRuleStmtTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index 96090cfb104c41..cf409d5fb0a26e 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -199,6 +199,7 @@ module.exports = [ "time-zone", "variables", "update", + "sql-block", ], sidebarDepth: 1, }, diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index f2fbb5f57480f1..ed0ba4f1d2cb04 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -200,6 +200,7 @@ module.exports = [ "time-zone", "variables", "update", + "sql-block", ], sidebarDepth: 1, }, diff --git a/docs/en/administrator-guide/block-rule/sql-block.md b/docs/en/administrator-guide/block-rule/sql-block.md new file mode 100644 index 00000000000000..54390a2f4a5d04 --- /dev/null +++ b/docs/en/administrator-guide/block-rule/sql-block.md @@ -0,0 +1,45 @@ +--- +{ +"title": "SQL Block Rule", +"language": "en" +} +--- + + + +# SQL Block Rule + +Support SQL block rule by user level, by regex way to deny specify SQL + +## Specific operation + +SQL block rule CRUD +- create SQL block rule + - user：For users whose rule is in effect, default means that all users are in effect. If both the specified user and the default rule are hit, the default rule takes precedence + - sql：Regex pattern，Special characters need to be translated + - sqlHash: Sql hash value, Used to match exactly, We print it in fe.audit.log + - enable：Whether to enable block rule +> CREATE SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","sqlHash":null,"enable"="true") +- show configured SQL block rules, or show all rules if you do not specify a rule name +> SHOW SQL_BLOCK_RULE [FOR RULE_NAME] +- alter SQL block rule，Allows changes user/sql/enable anyone +> ALTER SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","enable"="true") +- drop SQL block rule，Support multiple rules, separated by `,` +> DROP SQL_BLOCK_RULE test_rule1,test_rule2 diff --git a/docs/zh-CN/administrator-guide/block-rule/sql-block.md b/docs/zh-CN/administrator-guide/block-rule/sql-block.md new file mode 100644 index 00000000000000..3b64bed1672644 --- /dev/null +++ b/docs/zh-CN/administrator-guide/block-rule/sql-block.md @@ -0,0 +1,45 @@ +--- +{ +"title": "SQL黑名单", +"language": "zh-CN" +} +--- + + + +# SQL黑名单 + +支持按用户配置SQL黑名单，通过正则匹配的方式拒绝指定SQL + +## 具体操作 + +对SQL规则增删改查 +- 创建SQL阻止规则 + - user：规则生效的用户，default代表所有用户都生效，如果同时命中指定用户和default的规则，default规则优先 + - sql：匹配规则(基于正则匹配,特殊字符需要转译) + - sqlHash: sql hash值，用于完全匹配，我们会在`fe.audit.log`打印这个值 + - enable：是否开启阻止规则 +> CREATE SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","sqlHash":null,"enable"="true") +- 查看已配置的SQL阻止规则，不指定规则名则为查看所有规则 +> SHOW SQL_BLOCK_RULE [FOR RULE_NAME] +- 修改SQL阻止规则，允许对user/sql/enable等每一项进行修改 +> ALTER SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","enable"="true") +- 删除SQL阻止规则，支持多规则，以`,`隔开 +> DROP SQL_BLOCK_RULE test_rule1,test_rule2 diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 5b67d85d966217..4e31d4047539df 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -295,8 +295,7 @@ nonterminal StatementBase stmt, show_stmt, show_param, help_stmt, load_stmt, use_stmt, kill_stmt, drop_stmt, recover_stmt, grant_stmt, revoke_stmt, create_stmt, set_stmt, sync_stmt, cancel_stmt, cancel_param, delete_stmt, link_stmt, migrate_stmt, enter_stmt, transaction_stmt, unsupported_stmt, export_stmt, admin_stmt, truncate_stmt, import_columns_stmt, import_delete_on_stmt, import_sequence_stmt, import_where_stmt, install_plugin_stmt, uninstall_plugin_stmt, - import_preceding_filter_stmt, - create_sql_block_rule_stmt, alter_sql_block_rule_stmt, drop_sql_block_rule_stmt, show_sql_block_rule_stmt; + import_preceding_filter_stmt; nonterminal String transaction_label; nonterminal ImportColumnDesc import_column_desc; @@ -724,14 +723,6 @@ stmt ::= {: RESULT = stmt; :} | uninstall_plugin_stmt : stmt {: RESULT = stmt; :} - | create_sql_block_rule_stmt : stmt - {: RESULT = stmt; :} - | alter_sql_block_rule_stmt : stmt - {: RESULT = stmt; :} - | drop_sql_block_rule_stmt : stmt - {: RESULT = stmt; :} - | show_sql_block_rule_stmt : stmt - {: RESULT = stmt; :} | /* empty: query only has comments */ {: RESULT = new EmptyStmt(); @@ -836,6 +827,11 @@ alter_stmt ::= {: RESULT = new AlterRoutineLoadStmt(jobLabel, jobProperties, datasourceProperties); :} + | KW_ALTER KW_SQL_BLOCK_RULE STRING_LITERAL:ruleName + opt_properties:properties + {: + RESULT = new AlterSqlBlockRuleStmt(ruleName, properties); + :} ; opt_datasource_properties ::= @@ -1263,6 +1259,10 @@ create_stmt ::= {: RESULT = new CreateDataSyncJobStmt(jobName, db, channelDescList, binlog, properties); :} + | KW_CREATE KW_SQL_BLOCK_RULE ident:ruleName opt_properties:properties + {: + RESULT = new CreateSqlBlockRuleStmt(ruleName, properties); + :} ; channel_desc_list ::= @@ -1827,41 +1827,6 @@ show_create_routine_load_stmt ::= :} ; -// Sql block rule statement -create_sql_block_rule_stmt ::= - KW_CREATE KW_SQL_BLOCK_RULE STRING_LITERAL:ruleName - opt_properties:properties - {: - RESULT = new CreateSqlBlockRuleStmt(ruleName, properties); - :} - ; - -alter_sql_block_rule_stmt ::= - KW_ALTER KW_SQL_BLOCK_RULE STRING_LITERAL:ruleName - opt_properties:properties - {: - RESULT = new AlterSqlBlockRuleStmt(ruleName, properties); - :} - ; - -drop_sql_block_rule_stmt ::= - KW_DROP KW_SQL_BLOCK_RULE ident_list:ruleNames - {: - RESULT = new DropSqlBlockRuleStmt(ruleNames); - :} - ; - -show_sql_block_rule_stmt ::= - KW_SHOW KW_SQL_BLOCK_RULE KW_FOR ident:ruleName - {: - RESULT = new ShowSqlBlockRuleStmt(ruleName); - :} - | KW_SHOW KW_SQL_BLOCK_RULE - {: - RESULT = new ShowSqlBlockRuleStmt(null); - :} - ; - // Grant statement grant_stmt ::= KW_GRANT privilege_list:privs KW_ON tbl_pattern:tblPattern KW_TO user_identity:userId @@ -1999,6 +1964,10 @@ drop_stmt ::= {: RESULT = new DropEncryptKeyStmt(keyName); :} + | KW_DROP KW_SQL_BLOCK_RULE ident_list:ruleNames + {: + RESULT = new DropSqlBlockRuleStmt(ruleNames); + :} ; // Recover statement @@ -2466,6 +2435,14 @@ show_stmt ::= {: RESULT = stmt; :} + | KW_SHOW KW_SQL_BLOCK_RULE KW_FOR ident:ruleName + {: + RESULT = new ShowSqlBlockRuleStmt(ruleName); + :} + | KW_SHOW KW_SQL_BLOCK_RULE + {: + RESULT = new ShowSqlBlockRuleStmt(null); + :} ; show_param ::= diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java index 9dba978565c032..b8f1bb8e191e3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java @@ -17,16 +17,19 @@ package org.apache.doris.analysis; -import org.apache.doris.block.SqlBlockRule; +import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.commons.lang3.StringUtils; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.Optional; public class AlterSqlBlockRuleStmt extends DdlStmt { @@ -36,6 +39,8 @@ public class AlterSqlBlockRuleStmt extends DdlStmt { private String sql; + private String sqlHash; + private Boolean enable; private final Map properties; @@ -48,20 +53,27 @@ public AlterSqlBlockRuleStmt(String ruleName, Map properties) { @Override public void analyze(Analyzer analyzer) throws UserException { super.analyze(analyzer); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } // check properties - checkProperties(properties); + CreateSqlBlockRuleStmt.checkCommonProperties(properties); + setProperties(properties); } - private void checkProperties(Map properties) throws UserException { + private void setProperties(Map properties) throws UserException { this.user = properties.get(CreateSqlBlockRuleStmt.USER_PROPERTY); // if not default, need check whether user exist if (StringUtils.isNotEmpty(user) && !SqlBlockRule.DEFAULT_USER.equals(user)) { - List allUserIdents = Catalog.getCurrentCatalog().getAuth().getAllUserIdents(false).stream().map(UserIdentity::getUser).collect(Collectors.toList()); - if (!allUserIdents.contains(user)) { - throw new AnalysisException(user + " is not exist"); + boolean existUser = Catalog.getCurrentCatalog().getAuth().getTablePrivTable().doesUsernameExist(user); + if (!existUser) { + throw new AnalysisException(user + " does not exist"); } } this.sql = properties.get(CreateSqlBlockRuleStmt.SQL_PROPERTY); + this.sqlHash = properties.get(CreateSqlBlockRuleStmt.SQL_HASH_PROPERTY); + // allow null, represents no modification String enableStr = properties.get(CreateSqlBlockRuleStmt.ENABLE_PROPERTY); this.enable = StringUtils.isNotEmpty(enableStr) ? Boolean.parseBoolean(enableStr) : null; } @@ -81,4 +93,8 @@ public String getSql() { public Boolean getEnable() { return enable; } + + public String getSqlHash() { + return sqlHash; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java index 251318b23502ff..bfc16445055d6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java @@ -17,25 +17,27 @@ package org.apache.doris.analysis; -import org.apache.doris.block.SqlBlockRule; +import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableSet; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; /* Create sqlBlockRule statement syntax: - CREATE SQL_BLOCK_RULE LOAD NAME PROPERTIES + CREATE SQL_BLOCK_RULE `rule_name` PROPERTIES ( user = default, sql = select * from a, @@ -44,12 +46,12 @@ */ public class CreateSqlBlockRuleStmt extends DdlStmt { - public static final String NAME_PROPERTY = "name"; - public static final String USER_PROPERTY = "user"; public static final String SQL_PROPERTY = "sql"; + public static final String SQL_HASH_PROPERTY = "sqlHash"; + public static final String ENABLE_PROPERTY = "enable"; private final String ruleName; @@ -59,6 +61,8 @@ public class CreateSqlBlockRuleStmt extends DdlStmt { private String sql; + private String sqlHash; + // whether to use the rule private boolean enable; @@ -66,10 +70,10 @@ public class CreateSqlBlockRuleStmt extends DdlStmt { private static final String NAME_TYPE = "SQL BLOCK RULE NAME"; - private static final ImmutableSet PROPERTIES_SET = new ImmutableSet.Builder() - .add(NAME_PROPERTY) + public static final ImmutableSet PROPERTIES_SET = new ImmutableSet.Builder() .add(USER_PROPERTY) .add(SQL_PROPERTY) + .add(SQL_HASH_PROPERTY) .add(ENABLE_PROPERTY) .build(); @@ -82,28 +86,40 @@ public CreateSqlBlockRuleStmt(String ruleName, Map properties) { public void analyze(Analyzer analyzer) throws UserException { // check name FeNameFormat.checkCommonName(NAME_TYPE, ruleName); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } // check properties - checkProperties(properties); + CreateSqlBlockRuleStmt.checkCommonProperties(properties); + setProperties(properties); } - private void checkProperties(Map properties) throws UserException { - Optional optional = properties.keySet().stream().filter( - entity -> !PROPERTIES_SET.contains(entity)).findFirst(); - if (optional.isPresent()) { - throw new AnalysisException(optional.get() + " is invalid property"); - } + private void setProperties(Map properties) throws UserException { this.user = properties.get(USER_PROPERTY); // if not default, need check whether user exist if (!SqlBlockRule.DEFAULT_USER.equals(user)) { - List allUserIdents = Catalog.getCurrentCatalog().getAuth().getAllUserIdents(false).stream().map(UserIdentity::getUser).collect(Collectors.toList()); - if (!allUserIdents.contains(user)) { - throw new AnalysisException(user + " is not exist"); + boolean existUser = Catalog.getCurrentCatalog().getAuth().getTablePrivTable().doesUsernameExist(user); + if (!existUser) { + throw new AnalysisException(user + " does not exist"); } } this.sql = properties.get(SQL_PROPERTY); + this.sqlHash = properties.get(SQL_HASH_PROPERTY); this.enable = Util.getBooleanPropertyOrDefault(properties.get(ENABLE_PROPERTY), true, ENABLE_PROPERTY + " should be a boolean"); } + public static void checkCommonProperties(Map properties) throws UserException { + if (properties == null || properties.isEmpty()) { + throw new AnalysisException("Not set properties"); + } + Optional optional = properties.keySet().stream().filter( + entity -> !PROPERTIES_SET.contains(entity)).findFirst(); + if (optional.isPresent()) { + throw new AnalysisException(optional.get() + " is invalid property"); + } + } + public String getRuleName() { return ruleName; } @@ -116,6 +132,10 @@ public String getSql() { return sql; } + public String getSqlHash() { + return sqlHash; + } + public boolean isEnable() { return enable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java index 946b76e9145887..2c95e70df603fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java @@ -17,12 +17,28 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + import java.util.List; public class DropSqlBlockRuleStmt extends DdlStmt { private List ruleNames; + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + } + public DropSqlBlockRuleStmt(List ruleNames) { this.ruleNames = ruleNames; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java similarity index 80% rename from fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java rename to fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java index d2449f3dc99e72..2a89728019aa19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.block; +package org.apache.doris.blockrule; import org.apache.doris.analysis.AlterSqlBlockRuleStmt; import org.apache.doris.analysis.CreateSqlBlockRuleStmt; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; @@ -43,6 +44,9 @@ public class SqlBlockRule implements Writable { private String sql; + // sql md5 + private String sqlHash; + // whether to use the rule private Boolean enable; @@ -50,19 +54,20 @@ public SqlBlockRule(String name) { this.name = name; } - public SqlBlockRule(String name, String user, String sql, Boolean enable) { + public SqlBlockRule(String name, String user, String sql, String sqlHash, Boolean enable) { this.name = name; this.user = user; this.sql = sql; + this.sqlHash = sqlHash; this.enable = enable; } public static SqlBlockRule fromCreateStmt(CreateSqlBlockRuleStmt stmt) { - return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.isEnable()); + return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.getSqlHash(), stmt.isEnable()); } public static SqlBlockRule fromAlterStmt(AlterSqlBlockRuleStmt stmt) { - return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.getEnable()); + return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.getSqlHash(), stmt.getEnable()); } public String getName() { @@ -77,6 +82,10 @@ public String getSql() { return sql; } + public String getSqlHash() { + return sqlHash; + } + public Boolean getEnable() { return enable; } @@ -89,6 +98,10 @@ public void setSql(String sql) { this.sql = sql; } + public void setSqlHash(String sqlHash) { + this.sqlHash = sqlHash; + } + public void setEnable(Boolean enable) { this.enable = enable; } @@ -99,13 +112,11 @@ public List getShowInfo() { @Override public void write(DataOutput out) throws IOException { - Text.writeString(out, name); - Text.writeString(out, user); - Text.writeString(out, sql); - Text.writeString(out, String.valueOf(enable)); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } public static SqlBlockRule read(DataInput in) throws IOException { - return new SqlBlockRule(Text.readString(in), Text.readString(in), Text.readString(in), Boolean.valueOf(Text.readString(in))); + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, SqlBlockRule.class); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java similarity index 77% rename from fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java rename to fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index a08e4e9341cd84..6363fc4e9c7bf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/block/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.block; +package org.apache.doris.blockrule; import org.apache.doris.analysis.AlterSqlBlockRuleStmt; import org.apache.doris.analysis.CreateSqlBlockRuleStmt; @@ -28,12 +28,15 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; +import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; public class SqlBlockRuleMgr implements Writable { private static final Logger LOG = LogManager.getLogger(SqlBlockRuleMgr.class); @@ -55,6 +59,8 @@ public class SqlBlockRuleMgr implements Writable { private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); + private Map sqlPatternMap = Maps.newConcurrentMap(); + private void writeLock() { lock.writeLock().lock(); } @@ -83,24 +89,15 @@ public List get(ShowSqlBlockRuleStmt stmt) throws AnalysisExceptio } public void createSqlBlockRule(CreateSqlBlockRuleStmt stmt) throws UserException { - if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) - && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), - PrivPredicate.OPERATOR)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); - } - SqlBlockRule sqlBlockRule = SqlBlockRule.fromCreateStmt(stmt); - create(sqlBlockRule); - } - - private void create(SqlBlockRule sqlBlockRule) throws DdlException { writeLock(); try { + SqlBlockRule sqlBlockRule = SqlBlockRule.fromCreateStmt(stmt); String ruleName = sqlBlockRule.getName(); if (existRule(ruleName)) { throw new DdlException("the sql block rule " + ruleName + " already create"); } - Catalog.getCurrentCatalog().getEditLog().logCreateSqlBlockRule(sqlBlockRule); unprotectedAdd(sqlBlockRule); + Catalog.getCurrentCatalog().getEditLog().logCreateSqlBlockRule(sqlBlockRule); } finally { writeUnlock(); } @@ -111,19 +108,10 @@ public void replayCreate(SqlBlockRule sqlBlockRule) { LOG.info("replay create sql block rule: {}", sqlBlockRule); } - public void alterSqlBlockRule(AlterSqlBlockRuleStmt stmt) throws UserException { - if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) - && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), - PrivPredicate.OPERATOR)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); - } - SqlBlockRule sqlBlockRule = SqlBlockRule.fromAlterStmt(stmt); - alter(sqlBlockRule); - } - - private void alter(SqlBlockRule sqlBlockRule) throws DdlException { + public void alterSqlBlockRule(AlterSqlBlockRuleStmt stmt) throws DdlException { writeLock(); try { + SqlBlockRule sqlBlockRule = SqlBlockRule.fromAlterStmt(stmt); String ruleName = sqlBlockRule.getName(); if (!existRule(ruleName)) { throw new DdlException("the sql block rule " + ruleName + " not exist"); @@ -135,11 +123,14 @@ private void alter(SqlBlockRule sqlBlockRule) throws DdlException { if (StringUtils.isEmpty(sqlBlockRule.getSql())) { sqlBlockRule.setSql(originRule.getSql()); } + if (StringUtils.isEmpty(sqlBlockRule.getSqlHash())) { + sqlBlockRule.setSqlHash(originRule.getSqlHash()); + } if (sqlBlockRule.getEnable() == null) { sqlBlockRule.setEnable(originRule.getEnable()); } - Catalog.getCurrentCatalog().getEditLog().logAlterSqlBlockRule(sqlBlockRule); unprotectedUpdate(sqlBlockRule); + Catalog.getCurrentCatalog().getEditLog().logAlterSqlBlockRule(sqlBlockRule); } finally { writeUnlock(); } @@ -163,21 +154,16 @@ public void unprotectedAdd(SqlBlockRule sqlBlockRule) { List sqlBlockRules = userToSqlBlockRuleMap.getOrDefault(sqlBlockRule.getUser(), new ArrayList<>()); sqlBlockRules.add(sqlBlockRule); userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); - } - - public void dropSqlBlockRule(DropSqlBlockRuleStmt stmt) throws UserException { - if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) - && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), - PrivPredicate.OPERATOR)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + String sql = sqlBlockRule.getSql(); + if (StringUtils.isNotEmpty(sql)) { + sqlPatternMap.put(sql, Pattern.compile(sql)); } - List ruleNames = stmt.getRuleNames(); - drop(ruleNames); } - public void drop(List ruleNames) throws DdlException { + public void dropSqlBlockRule(DropSqlBlockRuleStmt stmt) throws DdlException { writeLock(); try { + List ruleNames = stmt.getRuleNames(); for (String ruleName : ruleNames) { if (!existRule(ruleName)) { throw new DdlException("the sql block rule " + ruleName + " not exist"); @@ -186,8 +172,8 @@ public void drop(List ruleNames) throws DdlException { if (sqlBlockRule == null) { continue; } - Catalog.getCurrentCatalog().getEditLog().logDropSqlBlockRule(sqlBlockRule); unprotectedDrop(sqlBlockRule); + Catalog.getCurrentCatalog().getEditLog().logDropSqlBlockRule(sqlBlockRule); } } finally { writeUnlock(); @@ -206,8 +192,33 @@ public void unprotectedDrop(SqlBlockRule sqlBlockRule) { userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); } - public Map> getUserToSqlBlockRuleMap() { - return userToSqlBlockRuleMap; + public void matchSql(String sql, String user) throws AnalysisException { + List defaultRules = userToSqlBlockRuleMap.getOrDefault(SqlBlockRule.DEFAULT_USER, new ArrayList<>()); + for (SqlBlockRule rule : defaultRules) { + Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); + matchSql(rule, sql, sqlPattern); + } + // match user rule + List userRules = userToSqlBlockRuleMap.getOrDefault(user, new ArrayList<>()); + for (SqlBlockRule rule : userRules) { + Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); + matchSql(rule, sql, sqlPattern); + } + } + + @VisibleForTesting + public static void matchSql(SqlBlockRule rule, String sql, Pattern sqlPattern) throws AnalysisException { + if (rule.getEnable() != null && rule.getEnable()) { + String sqlHash = rule.getSqlHash(); + if (sqlHash != null && sqlHash.equals(DigestUtils.md5Hex(sql))) { + MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); + throw new AnalysisException("sql match hash sql block rule: " + rule.getName()); + } + if (sqlPattern != null && sqlPattern.matcher(sql).find()) { + MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); + throw new AnalysisException("sql match regex sql block rule: " + rule.getName()); + } + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 67e052a5aef281..8287cb99307212 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -85,7 +85,7 @@ import org.apache.doris.analysis.UserDesc; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupHandler; -import org.apache.doris.block.SqlBlockRuleMgr; +import org.apache.doris.blockrule.SqlBlockRuleMgr; import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.Database.DbState; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; @@ -2186,7 +2186,7 @@ public long saveSmallFiles(CountingDataOutputStream dos, long checksum) throws I } public long saveSqlBlockRule(DataOutputStream out, long checksum) throws IOException { - Catalog.getCurrentCatalog().getSqlBlocklistMgr().write(out); + Catalog.getCurrentCatalog().getSqlBlockRuleMgr().write(out); return checksum; } @@ -4880,7 +4880,7 @@ public RoutineLoadManager getRoutineLoadManager() { return routineLoadManager; } - public SqlBlockRuleMgr getSqlBlocklistMgr() { + public SqlBlockRuleMgr getSqlBlockRuleMgr() { return sqlBlockRuleMgr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 9f9b3b9e34c9eb..e9469ffd40ad76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -24,6 +24,7 @@ import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; import org.apache.doris.backup.RestoreJob; +import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.BrokerMgr; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.EncryptKey; @@ -638,6 +639,21 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_CREATE_SQL_BLOCK_RULE: { + data = SqlBlockRule.read(in); + isRead = true; + break; + } + case OperationType.OP_ALTER_SQL_BLOCK_RULE: { + data = SqlBlockRule.read(in); + isRead = true; + break; + } + case OperationType.OP_DROP_SQL_BLOCK_RULE: { + data = SqlBlockRule.read(in); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 941857a738d6b7..968a02433536ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -303,8 +303,8 @@ public Long getValue() { "total error rows of routine load"); PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_ROUTINE_LOAD_ERROR_ROWS); - COUNTER_HIT_SQL_BLOCK_RULE = new LongCounterMetric("counter_hit_sql_blacklist", MetricUnit.ROWS, - "total hit sql blacklist query"); + COUNTER_HIT_SQL_BLOCK_RULE = new LongCounterMetric("counter_hit_sql_block_rule", MetricUnit.ROWS, + "total hit sql block rule query"); PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_HIT_SQL_BLOCK_RULE); // 3. histogram HISTO_QUERY_LATENCY = METRIC_REGISTER.histogram(MetricRegistry.name("query", "latency", "ms")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java index 8c9d7499cf018e..209ed36742ed2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java @@ -1333,7 +1333,7 @@ private void getUserAuthInfo(List> userAuthInfos, UserIdentity user userAuthInfos.add(userAuthInfo); } - public Set getAllUserIdents(boolean includeEntrySetByResolver) { + private Set getAllUserIdents(boolean includeEntrySetByResolver) { Set userIdents = Sets.newHashSet(); for (PrivEntry entry : userPrivTable.entries) { if (!includeEntrySetByResolver && entry.isSetByDomainResolver()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index f2b7b3550354ee..a1a538dae54558 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -26,7 +26,7 @@ import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; import org.apache.doris.backup.RestoreJob; -import org.apache.doris.block.SqlBlockRule; +import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.BrokerMgr; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; @@ -846,17 +846,17 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { } case OperationType.OP_CREATE_SQL_BLOCK_RULE: { SqlBlockRule rule = (SqlBlockRule) journal.getData(); - catalog.getSqlBlocklistMgr().replayCreate(rule); + catalog.getSqlBlockRuleMgr().replayCreate(rule); break; } case OperationType.OP_ALTER_SQL_BLOCK_RULE: { SqlBlockRule rule = (SqlBlockRule) journal.getData(); - catalog.getSqlBlocklistMgr().replayAlter(rule); + catalog.getSqlBlockRuleMgr().replayAlter(rule); break; } case OperationType.OP_DROP_SQL_BLOCK_RULE: { SqlBlockRule rule = (SqlBlockRule) journal.getData(); - catalog.getSqlBlocklistMgr().replayDrop(rule); + catalog.getSqlBlockRuleMgr().replayDrop(rule); break; } default: { diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java index af789184615106..0054e7eaadf040 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java @@ -79,6 +79,8 @@ public enum EventType { public String stmt = ""; @AuditField(value = "CpuTimeMS") public long cpuTimeMs = -1; + @AuditField(value = "SqlHash") + public String sqlHash = ""; public static class AuditEventBuilder { @@ -171,6 +173,11 @@ public AuditEventBuilder setStmt(String stmt) { return this; } + public AuditEventBuilder setSqlHash(String sqlHash) { + auditEvent.sqlHash = sqlHash; + return this; + } + public AuditEvent build() { return this.auditEvent; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 6ce8906033aad7..967f28431f57fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -53,6 +53,7 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -144,9 +145,9 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue ctx.getAuditEventBuilder().setFeIp(FrontendOptions.getLocalHostAddress()); - // We put origin query stmt at the end of audit log, for parsing the log more convenient. + String sql; if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { - ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql()); + sql = parsedStmt.toSql(); } else { if (parsedStmt instanceof InsertStmt && ((InsertStmt)parsedStmt).isValuesOrConstantSelect()) { // INSERT INTO VALUES may be very long, so we only log at most 1K bytes. @@ -155,8 +156,11 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue } else { ctx.getAuditEventBuilder().setStmt(origStmt); } + sql = origStmt; } - + ctx.getAuditEventBuilder().setSqlHash(DigestUtils.md5Hex(sql)); + // We put origin query stmt at the end of audit log, for parsing the log more convenient. + ctx.getAuditEventBuilder().setStmt(sql); Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index c5c3e996a10f0c..cbe1fb50e4c3c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -267,11 +267,11 @@ public static void execute(Catalog catalog, DdlStmt ddlStmt) throws Exception { } else if (ddlStmt instanceof AdminCleanTrashStmt) { catalog.cleanTrash((AdminCleanTrashStmt) ddlStmt); } else if (ddlStmt instanceof CreateSqlBlockRuleStmt) { - catalog.getSqlBlocklistMgr().createSqlBlockRule((CreateSqlBlockRuleStmt) ddlStmt); + catalog.getSqlBlockRuleMgr().createSqlBlockRule((CreateSqlBlockRuleStmt) ddlStmt); } else if (ddlStmt instanceof AlterSqlBlockRuleStmt) { - catalog.getSqlBlocklistMgr().alterSqlBlockRule((AlterSqlBlockRuleStmt) ddlStmt); + catalog.getSqlBlockRuleMgr().alterSqlBlockRule((AlterSqlBlockRuleStmt) ddlStmt); } else if (ddlStmt instanceof DropSqlBlockRuleStmt) { - catalog.getSqlBlocklistMgr().dropSqlBlockRule((DropSqlBlockRuleStmt) ddlStmt); + catalog.getSqlBlockRuleMgr().dropSqlBlockRule((DropSqlBlockRuleStmt) ddlStmt); } else { throw new DdlException("Unknown statement."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 50dd526a79d542..8aae2e18eb2d57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -86,7 +86,7 @@ import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; import org.apache.doris.backup.RestoreJob; -import org.apache.doris.block.SqlBlockRule; +import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.BrokerMgr; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; @@ -2081,6 +2081,7 @@ private void handleShowCreateRoutineLoad() throws AnalysisException { resultSet = new ShowResultSet(showCreateRoutineLoadStmt.getMetaData(), rows); } +<<<<<<< HEAD private void handleAdminShowDataSkew() throws AnalysisException { AdminShowDataSkewStmt showStmt = (AdminShowDataSkewStmt) stmt; List> results; @@ -2095,7 +2096,7 @@ private void handleAdminShowDataSkew() throws AnalysisException { public void handleShowSqlBlockRule() throws AnalysisException { ShowSqlBlockRuleStmt showStmt = (ShowSqlBlockRuleStmt) stmt; List> rows = Lists.newArrayList(); - List sqlBlockRules = Catalog.getCurrentCatalog().getSqlBlocklistMgr().get(showStmt); + List sqlBlockRules = Catalog.getCurrentCatalog().getSqlBlockRuleMgr().get(showStmt); sqlBlockRules.forEach(rule -> rows.add(rule.getShowInfo())); resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 860de0147d50c4..9efa175866d441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -45,7 +45,6 @@ import org.apache.doris.analysis.TransactionStmt; import org.apache.doris.analysis.UnsupportedStmt; import org.apache.doris.analysis.UseStmt; -import org.apache.doris.block.SqlBlockRule; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; @@ -109,7 +108,6 @@ import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -123,7 +121,6 @@ import java.io.IOException; import java.io.StringReader; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -131,7 +128,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Pattern; import java.util.stream.Collectors; // Do one COM_QUERY process. @@ -328,11 +324,9 @@ public void execute(TUniqueId queryId) throws Exception { } if (parsedStmt instanceof QueryStmt) { - boolean enableSqlBlock = Config.enable_sql_block; - if (enableSqlBlock) { - QueryStmt queryStmt = (QueryStmt) this.parsedStmt; - matchSql(queryStmt); - } + QueryStmt queryStmt = (QueryStmt) this.parsedStmt; + // match sql + matchSql(queryStmt); context.getState().setIsQuery(true); MetricRepo.COUNTER_QUERY_BEGIN.increase(1L); int retryTime = Config.max_query_retry_time; @@ -1471,31 +1465,11 @@ private List exprToType(List exprs) { } private void matchSql(QueryStmt queryStmt) throws AnalysisException { - Map> userToSqlBlockRuleMap = Catalog.getCurrentCatalog().getSqlBlocklistMgr().getUserToSqlBlockRuleMap(); - // match default rule String sql = queryStmt.getOrigStmt().originStmt; - List defaultRules = userToSqlBlockRuleMap.getOrDefault(SqlBlockRule.DEFAULT_USER, new ArrayList<>()); - for (SqlBlockRule rule : defaultRules) { - matchSql(rule, sql); - } - // match user rule String user = context.getUserIdentity().getUser(); - List userRules = userToSqlBlockRuleMap.getOrDefault(user, new ArrayList<>()); - for (SqlBlockRule rule : userRules) { - matchSql(rule, sql); - } + Catalog.getCurrentCatalog().getSqlBlockRuleMgr().matchSql(sql, user); } - @VisibleForTesting - public static void matchSql(SqlBlockRule rule, String sql) throws AnalysisException { - if (rule.getEnable() != null && rule.getEnable()) { - String sqlPattern = rule.getSql(); - Pattern pattern = Pattern.compile(sqlPattern); - if (pattern.matcher(sql).find()) { - MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); - throw new AnalysisException("query match sql block rule: " + rule.getName()); - } - } - } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterSqlBlockRuleStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterSqlBlockRuleStmtTest.java new file mode 100644 index 00000000000000..1ed141739a111b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterSqlBlockRuleStmtTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.blockrule.SqlBlockRule; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.mysql.privilege.PaloAuth; +import org.apache.doris.qe.ConnectContext; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import mockit.Mocked; + +public class AlterSqlBlockRuleStmtTest { + + private Analyzer analyzer; + + @Mocked + private PaloAuth auth; + + @Mocked + private ConnectContext ctx; + + @Before + public void setUp() { + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); + } + + @Test + public void testNormal() throws UserException { + Map properties = new HashMap<>(); + properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); + properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "false"); + AlterSqlBlockRuleStmt stmt = new AlterSqlBlockRuleStmt("test_rule", properties); + stmt.analyze(analyzer); + SqlBlockRule rule = SqlBlockRule.fromAlterStmt(stmt); + Assert.assertEquals(false, rule.getEnable()); + Assert.assertEquals("select \\* from test_table", rule.getSql()); + Assert.assertEquals("test_rule", rule.getName()); + } + + @Test(expected = AnalysisException.class) + public void testNoProps() throws UserException { + Map properties = new HashMap<>(); + AlterSqlBlockRuleStmt stmt = new AlterSqlBlockRuleStmt("test_rule", properties); + stmt.analyze(analyzer); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java new file mode 100644 index 00000000000000..1183fb8afd2eca --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.blockrule.SqlBlockRule; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.mysql.privilege.PaloAuth; +import org.apache.doris.qe.ConnectContext; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import mockit.Mocked; + +public class CreateSqlBlockRuleStmtTest { + + private Analyzer analyzer; + + @Mocked + private PaloAuth auth; + + @Mocked + private ConnectContext ctx; + + @Before + public void setUp() { + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); + } + + @Test + public void testNormal() throws UserException { + Map properties = new HashMap<>(); + properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); + properties.put(CreateSqlBlockRuleStmt.USER_PROPERTY, SqlBlockRule.DEFAULT_USER); + properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); + CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); + stmt.analyze(analyzer); + SqlBlockRule rule = SqlBlockRule.fromCreateStmt(stmt); + Assert.assertEquals(true, rule.getEnable()); + Assert.assertEquals("select \\* from test_table", rule.getSql()); + Assert.assertEquals("test_rule", rule.getName()); + } + + @Test(expected = AnalysisException.class) + public void testNoProps() throws UserException { + Map properties = new HashMap<>(); + properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); + properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); + CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); + stmt.analyze(analyzer); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java new file mode 100644 index 00000000000000..df769f92651679 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.blockrule; + +import org.apache.doris.analysis.AccessTestUtil; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateSqlBlockRuleStmt; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.qe.ConnectContext; + +import org.apache.commons.codec.digest.DigestUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import mockit.Mocked; + +public class SqlBlockRuleMgrTest { + + private Analyzer analyzer; + + @Before + public void setUp() { + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MetricRepo.init(); + } + + + @Test(expected = AnalysisException.class) + public void testRegexMatchSql() throws AnalysisException { + String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", "default", ".* join .*", null, true); + Pattern sqlPattern = Pattern.compile(sqlRule.getSql()); + SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlPattern); + } + + @Test(expected = AnalysisException.class) + public void testHashMatchSql() throws AnalysisException { + String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; + String hashSql = DigestUtils.md5Hex(sql); + System.out.println(hashSql); + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", "default", null, hashSql, true); + SqlBlockRuleMgr.matchSql(sqlRule, sql, null); + } + + @Test + public void testNormalCreate(@Mocked ConnectContext connectContext, @Mocked Catalog catalog) throws UserException { + SqlBlockRuleMgr sqlBlockRuleMgr = new SqlBlockRuleMgr(); + Map properties = new HashMap<>(); + properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); + properties.put(CreateSqlBlockRuleStmt.USER_PROPERTY, SqlBlockRule.DEFAULT_USER); + properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); + CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); + stmt.analyze(analyzer); + sqlBlockRuleMgr.createSqlBlockRule(stmt); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java index a958bb53188094..51cc6705e46a97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java @@ -29,6 +29,7 @@ import org.apache.doris.analysis.ShowDbStmt; import org.apache.doris.analysis.ShowEnginesStmt; import org.apache.doris.analysis.ShowProcedureStmt; +import org.apache.doris.analysis.ShowSqlBlockRuleStmt; import org.apache.doris.analysis.ShowTableStmt; import org.apache.doris.analysis.ShowVariablesStmt; import org.apache.doris.analysis.ShowViewStmt; @@ -589,4 +590,16 @@ public void testHelp() throws AnalysisException, IOException, UserException { Assert.assertFalse(resultSet.next()); } + + @Test + public void testShowSqlBlockRule() throws AnalysisException { + ShowSqlBlockRuleStmt stmt = new ShowSqlBlockRuleStmt("test_rule"); + ShowExecutor executor = new ShowExecutor(ctx, stmt); + ShowResultSet resultSet = executor.execute(); + Assert.assertEquals(4, resultSet.getMetaData().getColumnCount()); + Assert.assertEquals("Name", resultSet.getMetaData().getColumn(0).getName()); + Assert.assertEquals("User", resultSet.getMetaData().getColumn(1).getName()); + Assert.assertEquals("Sql", resultSet.getMetaData().getColumn(2).getName()); + Assert.assertEquals("Enable", resultSet.getMetaData().getColumn(3).getName()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 93045f4101b291..506b08ca9da11e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -30,9 +30,7 @@ import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UseStmt; -import org.apache.doris.block.SqlBlockRule; import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.RuntimeProfile; @@ -734,18 +732,5 @@ public void testUseFail(@Mocked UseStmt useStmt, @Mocked SqlParser parser) throw Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); } - - @Test(expected = AnalysisException.class) - public void testMatchSql() throws AnalysisException { -// String sql = "select * from test_table join limit 10"; -// SqlBlockRule rule = new SqlBlockRule("test_rule", "default", "select \\* from .*", true); -// StmtExecutor.matchSql(rule, sql); -// String sql1 = "select * from test_table limit 5"; -// SqlBlockRule rule1 = new SqlBlockRule("test_rule1", "default", "select \\* from test_table limit [1-9]", true); -// StmtExecutor.matchSql(rule1, sql1); - String sql2 = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; - SqlBlockRule rule2 = new SqlBlockRule("test_rule1", "default", ".* join .*", true); - StmtExecutor.matchSql(rule2, sql2); - } } From c8ffc854ffbbdb0f5da07cebdd13530e5a6e5634 Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 20 Jul 2021 21:26:01 +0800 Subject: [PATCH 03/14] MOD: move user to UserProperty --- .../block-rule/sql-block.md | 32 +++++++++++----- .../block-rule/sql-block.md | 34 ++++++++++++----- .../doris/analysis/AlterSqlBlockRuleStmt.java | 27 +++++-------- .../analysis/CreateSqlBlockRuleStmt.java | 32 ++++++---------- .../apache/doris/blockrule/SqlBlockRule.java | 32 ++++++++-------- .../doris/blockrule/SqlBlockRuleMgr.java | 38 ++++++++----------- .../doris/mysql/privilege/PaloAuth.java | 9 +++++ .../doris/mysql/privilege/UserProperty.java | 37 ++++++++++++++++++ .../mysql/privilege/UserPropertyMgr.java | 8 ++++ .../org/apache/doris/qe/ConnectProcessor.java | 9 ++++- .../org/apache/doris/qe/StmtExecutor.java | 10 ----- .../analysis/CreateSqlBlockRuleStmtTest.java | 1 - .../doris/blockrule/SqlBlockRuleMgrTest.java | 5 +-- 13 files changed, 164 insertions(+), 110 deletions(-) diff --git a/docs/en/administrator-guide/block-rule/sql-block.md b/docs/en/administrator-guide/block-rule/sql-block.md index 54390a2f4a5d04..091181426763cf 100644 --- a/docs/en/administrator-guide/block-rule/sql-block.md +++ b/docs/en/administrator-guide/block-rule/sql-block.md @@ -28,18 +28,32 @@ under the License. Support SQL block rule by user level, by regex way to deny specify SQL -## Specific operation +## Rule SQL block rule CRUD - create SQL block rule - - user：For users whose rule is in effect, default means that all users are in effect. If both the specified user and the default rule are hit, the default rule takes precedence - sql：Regex pattern，Special characters need to be translated - - sqlHash: Sql hash value, Used to match exactly, We print it in fe.audit.log - - enable：Whether to enable block rule -> CREATE SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","sqlHash":null,"enable"="true") + - sqlHash: Sql hash value, Used to match exactly, We print it in fe.audit.log + - global: Whether global(all users)is in effect, false by default + - enable：Whether to enable block rule，true by default +``` +CREATE SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","sqlHash":null,"global"="false","enable"="true") +``` - show configured SQL block rules, or show all rules if you do not specify a rule name -> SHOW SQL_BLOCK_RULE [FOR RULE_NAME] -- alter SQL block rule，Allows changes user/sql/enable anyone -> ALTER SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","enable"="true") +``` +SHOW SQL_BLOCK_RULE [FOR RULE_NAME] +``` +- alter SQL block rule，Allows changes sql/global/enable anyone +``` +ALTER SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","enable"="true") +``` - drop SQL block rule，Support multiple rules, separated by `,` -> DROP SQL_BLOCK_RULE test_rule1,test_rule2 +``` +DROP SQL_BLOCK_RULE test_rule1,test_rule2 +``` + +## User bind rules +If global=false is configured, the rules binding for the specified user needs to be configured, with multiple rules separated by ', ' +``` +SET PROPERTY FOR 'jack' 'bind_sql_block_rules' = 'test_rule1,test_rule2' +``` diff --git a/docs/zh-CN/administrator-guide/block-rule/sql-block.md b/docs/zh-CN/administrator-guide/block-rule/sql-block.md index 3b64bed1672644..48a113737b15e8 100644 --- a/docs/zh-CN/administrator-guide/block-rule/sql-block.md +++ b/docs/zh-CN/administrator-guide/block-rule/sql-block.md @@ -28,18 +28,32 @@ under the License. 支持按用户配置SQL黑名单，通过正则匹配的方式拒绝指定SQL -## 具体操作 +## 规则 对SQL规则增删改查 - 创建SQL阻止规则 - - user：规则生效的用户，default代表所有用户都生效，如果同时命中指定用户和default的规则，default规则优先 - - sql：匹配规则(基于正则匹配,特殊字符需要转译) - - sqlHash: sql hash值，用于完全匹配，我们会在`fe.audit.log`打印这个值 - - enable：是否开启阻止规则 -> CREATE SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","sqlHash":null,"enable"="true") + - sql：匹配规则(基于正则匹配,特殊字符需要转译)，可选 + - sqlHash: sql hash值，用于完全匹配，我们会在`fe.audit.log`打印这个值，可选 + - global：是否全局(所有用户)生效，默认为false + - enable：是否开启阻止规则，默认为true +``` +CREATE SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","sqlHash":null,"enable"="true") +``` - 查看已配置的SQL阻止规则，不指定规则名则为查看所有规则 -> SHOW SQL_BLOCK_RULE [FOR RULE_NAME] -- 修改SQL阻止规则，允许对user/sql/enable等每一项进行修改 -> ALTER SQL_BLOCK_RULE test_rule PROPERTIES("user"="default","sql"="select \\* from test_table","enable"="true") +``` +SHOW SQL_BLOCK_RULE [FOR RULE_NAME] +``` +- 修改SQL阻止规则，允许对sql/global/enable等每一项进行修改 +``` +ALTER SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","enable"="true") +``` - 删除SQL阻止规则，支持多规则，以`,`隔开 -> DROP SQL_BLOCK_RULE test_rule1,test_rule2 +``` +DROP SQL_BLOCK_RULE test_rule1,test_rule2 +``` + +## 用户规则绑定 +如果配置global=false，则需要配置指定用户的规则绑定，多个规则使用`,`分隔 +``` +SET PROPERTY FOR 'jack' 'bind_sql_block_rules' = 'test_rule1,test_rule2' +``` \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java index b8f1bb8e191e3c..51ba67e9630f1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java @@ -17,9 +17,7 @@ package org.apache.doris.analysis; -import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -29,18 +27,17 @@ import org.apache.commons.lang3.StringUtils; import java.util.Map; -import java.util.Optional; public class AlterSqlBlockRuleStmt extends DdlStmt { private final String ruleName; - private String user; - private String sql; private String sqlHash; + private Boolean global; + private Boolean enable; private final Map properties; @@ -62,18 +59,12 @@ public void analyze(Analyzer analyzer) throws UserException { setProperties(properties); } - private void setProperties(Map properties) throws UserException { - this.user = properties.get(CreateSqlBlockRuleStmt.USER_PROPERTY); - // if not default, need check whether user exist - if (StringUtils.isNotEmpty(user) && !SqlBlockRule.DEFAULT_USER.equals(user)) { - boolean existUser = Catalog.getCurrentCatalog().getAuth().getTablePrivTable().doesUsernameExist(user); - if (!existUser) { - throw new AnalysisException(user + " does not exist"); - } - } + private void setProperties(Map properties) { this.sql = properties.get(CreateSqlBlockRuleStmt.SQL_PROPERTY); this.sqlHash = properties.get(CreateSqlBlockRuleStmt.SQL_HASH_PROPERTY); // allow null, represents no modification + String globalStr = properties.get(CreateSqlBlockRuleStmt.GLOBAL_PROPERTY); + this.global = StringUtils.isNotEmpty(globalStr) ? Boolean.parseBoolean(globalStr) : null; String enableStr = properties.get(CreateSqlBlockRuleStmt.ENABLE_PROPERTY); this.enable = StringUtils.isNotEmpty(enableStr) ? Boolean.parseBoolean(enableStr) : null; } @@ -82,14 +73,14 @@ public String getRuleName() { return ruleName; } - public String getUser() { - return user; - } - public String getSql() { return sql; } + public Boolean getGlobal() { + return global; + } + public Boolean getEnable() { return enable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java index bfc16445055d6c..3e2f84a0149d50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java @@ -17,7 +17,6 @@ package org.apache.doris.analysis; -import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; @@ -39,30 +38,30 @@ syntax: CREATE SQL_BLOCK_RULE `rule_name` PROPERTIES ( - user = default, sql = select * from a, + global = false enable = true ) */ public class CreateSqlBlockRuleStmt extends DdlStmt { - public static final String USER_PROPERTY = "user"; - public static final String SQL_PROPERTY = "sql"; public static final String SQL_HASH_PROPERTY = "sqlHash"; + public static final String GLOBAL_PROPERTY = "global"; + public static final String ENABLE_PROPERTY = "enable"; private final String ruleName; - // default stands for all users - private String user; - private String sql; private String sqlHash; + // whether effective global + private boolean global; + // whether to use the rule private boolean enable; @@ -71,9 +70,9 @@ public class CreateSqlBlockRuleStmt extends DdlStmt { private static final String NAME_TYPE = "SQL BLOCK RULE NAME"; public static final ImmutableSet PROPERTIES_SET = new ImmutableSet.Builder() - .add(USER_PROPERTY) .add(SQL_PROPERTY) .add(SQL_HASH_PROPERTY) + .add(GLOBAL_PROPERTY) .add(ENABLE_PROPERTY) .build(); @@ -96,16 +95,9 @@ public void analyze(Analyzer analyzer) throws UserException { } private void setProperties(Map properties) throws UserException { - this.user = properties.get(USER_PROPERTY); - // if not default, need check whether user exist - if (!SqlBlockRule.DEFAULT_USER.equals(user)) { - boolean existUser = Catalog.getCurrentCatalog().getAuth().getTablePrivTable().doesUsernameExist(user); - if (!existUser) { - throw new AnalysisException(user + " does not exist"); - } - } this.sql = properties.get(SQL_PROPERTY); this.sqlHash = properties.get(SQL_HASH_PROPERTY); + this.global = Util.getBooleanPropertyOrDefault(properties.get(GLOBAL_PROPERTY), false, GLOBAL_PROPERTY + " should be a boolean"); this.enable = Util.getBooleanPropertyOrDefault(properties.get(ENABLE_PROPERTY), true, ENABLE_PROPERTY + " should be a boolean"); } @@ -124,10 +116,6 @@ public String getRuleName() { return ruleName; } - public String getUser() { - return user; - } - public String getSql() { return sql; } @@ -136,6 +124,10 @@ public String getSqlHash() { return sqlHash; } + public boolean isGlobal() { + return global; + } + public boolean isEnable() { return enable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java index 2a89728019aa19..a11ffc3e3fb65e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java @@ -39,14 +39,14 @@ public class SqlBlockRule implements Writable { // the rule name, cluster unique private String name; - // default stands for all users - private String user; - private String sql; // sql md5 private String sqlHash; + // whether effective global + private Boolean global; + // whether to use the rule private Boolean enable; @@ -54,30 +54,26 @@ public SqlBlockRule(String name) { this.name = name; } - public SqlBlockRule(String name, String user, String sql, String sqlHash, Boolean enable) { + public SqlBlockRule(String name, String sql, String sqlHash, Boolean global, Boolean enable) { this.name = name; - this.user = user; this.sql = sql; this.sqlHash = sqlHash; + this.global = global; this.enable = enable; } public static SqlBlockRule fromCreateStmt(CreateSqlBlockRuleStmt stmt) { - return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.getSqlHash(), stmt.isEnable()); + return new SqlBlockRule(stmt.getRuleName(), stmt.getSql(), stmt.getSqlHash(), stmt.isGlobal(), stmt.isEnable()); } public static SqlBlockRule fromAlterStmt(AlterSqlBlockRuleStmt stmt) { - return new SqlBlockRule(stmt.getRuleName(), stmt.getUser(), stmt.getSql(), stmt.getSqlHash(), stmt.getEnable()); + return new SqlBlockRule(stmt.getRuleName(), stmt.getSql(), stmt.getSqlHash(), stmt.getGlobal(), stmt.getEnable()); } public String getName() { return name; } - public String getUser() { - return user; - } - public String getSql() { return sql; } @@ -86,12 +82,12 @@ public String getSqlHash() { return sqlHash; } - public Boolean getEnable() { - return enable; + public Boolean getGlobal() { + return global; } - public void setUser(String user) { - this.user = user; + public Boolean getEnable() { + return enable; } public void setSql(String sql) { @@ -102,12 +98,16 @@ public void setSqlHash(String sqlHash) { this.sqlHash = sqlHash; } + public void setGlobal(Boolean global) { + this.global = global; + } + public void setEnable(Boolean enable) { this.enable = enable; } public List getShowInfo() { - return Lists.newArrayList(this.name, this.user, this.sql, String.valueOf(this.enable)); + return Lists.newArrayList(this.name, this.sql, String.valueOf(this.enable)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index 6363fc4e9c7bf5..94179442d11b79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -44,19 +44,17 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class SqlBlockRuleMgr implements Writable { private static final Logger LOG = LogManager.getLogger(SqlBlockRuleMgr.class); private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); - private Map> userToSqlBlockRuleMap = Maps.newConcurrentMap(); - private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); private Map sqlPatternMap = Maps.newConcurrentMap(); @@ -117,15 +115,15 @@ public void alterSqlBlockRule(AlterSqlBlockRuleStmt stmt) throws DdlException { throw new DdlException("the sql block rule " + ruleName + " not exist"); } SqlBlockRule originRule = nameToSqlBlockRuleMap.get(ruleName); - if (StringUtils.isEmpty(sqlBlockRule.getUser())) { - sqlBlockRule.setUser(originRule.getUser()); - } if (StringUtils.isEmpty(sqlBlockRule.getSql())) { sqlBlockRule.setSql(originRule.getSql()); } if (StringUtils.isEmpty(sqlBlockRule.getSqlHash())) { sqlBlockRule.setSqlHash(originRule.getSqlHash()); } + if (sqlBlockRule.getGlobal() == null) { + sqlBlockRule.setGlobal(originRule.getGlobal()); + } if (sqlBlockRule.getEnable() == null) { sqlBlockRule.setEnable(originRule.getEnable()); } @@ -143,17 +141,10 @@ public void replayAlter(SqlBlockRule sqlBlockRule) { public void unprotectedUpdate(SqlBlockRule sqlBlockRule) { nameToSqlBlockRuleMap.put(sqlBlockRule.getName(), sqlBlockRule); - List sqlBlockRules = userToSqlBlockRuleMap.getOrDefault(sqlBlockRule.getUser(), new ArrayList<>()); - sqlBlockRules.removeIf(rule -> sqlBlockRule.getName().equals(rule.getName())); - sqlBlockRules.add(sqlBlockRule); - userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); } public void unprotectedAdd(SqlBlockRule sqlBlockRule) { nameToSqlBlockRuleMap.put(sqlBlockRule.getName(), sqlBlockRule); - List sqlBlockRules = userToSqlBlockRuleMap.getOrDefault(sqlBlockRule.getUser(), new ArrayList<>()); - sqlBlockRules.add(sqlBlockRule); - userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); String sql = sqlBlockRule.getSql(); if (StringUtils.isNotEmpty(sql)) { sqlPatternMap.put(sql, Pattern.compile(sql)); @@ -187,22 +178,25 @@ public void replayDrop(SqlBlockRule sqlBlockRule) { public void unprotectedDrop(SqlBlockRule sqlBlockRule) { nameToSqlBlockRuleMap.remove(sqlBlockRule.getName()); - List sqlBlockRules = userToSqlBlockRuleMap.get(sqlBlockRule.getUser()); - sqlBlockRules.removeIf(rule -> sqlBlockRule.getName().equals(rule.getName())); - userToSqlBlockRuleMap.put(sqlBlockRule.getUser(), sqlBlockRules); + // todo: remove UserProperty } public void matchSql(String sql, String user) throws AnalysisException { - List defaultRules = userToSqlBlockRuleMap.getOrDefault(SqlBlockRule.DEFAULT_USER, new ArrayList<>()); - for (SqlBlockRule rule : defaultRules) { + // match global rule + List globalRules = nameToSqlBlockRuleMap.values().stream().filter(SqlBlockRule::getGlobal).collect(Collectors.toList()); + for (SqlBlockRule rule : globalRules) { Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); matchSql(rule, sql, sqlPattern); } // match user rule - List userRules = userToSqlBlockRuleMap.getOrDefault(user, new ArrayList<>()); - for (SqlBlockRule rule : userRules) { - Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); - matchSql(rule, sql, sqlPattern); + String binSqlBlockRules = Catalog.getCurrentCatalog().getAuth().getBindSqlBlockRules(user); + if (StringUtils.isNotEmpty(binSqlBlockRules)) { + String[] split = binSqlBlockRules.split(","); + for (String ruleName : split) { + SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName); + Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); + matchSql(rule, sql, sqlPattern); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java index 209ed36742ed2f..11aaf7dd9bef68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java @@ -1134,6 +1134,15 @@ public long getMaxQueryInstances(String qualifiedUser) { } } + public String getBindSqlBlockRules(String qualifiedUser) { + readLock(); + try { + return propertyMgr.getBindSqlBlockRules(qualifiedUser); + } finally { + readUnlock(); + } + } + public void getAllDomains(Set allDomains) { readLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index 57578aeb1e2646..49784c1501d6c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -66,6 +66,7 @@ public class UserProperty implements Writable { private static final String PROP_QUOTA = "quota"; private static final String PROP_DEFAULT_LOAD_CLUSTER = "default_load_cluster"; private static final String PROP_LOAD_CLUSTER = "load_cluster"; + private static final String PROP_BIND_SQL_BLOCK_RULES = "bind_sql_block_rules"; // for system user public static final Set ADVANCED_PROPERTIES = Sets.newHashSet(); @@ -89,6 +90,18 @@ public class UserProperty implements Writable { */ private WhiteList whiteList = new WhiteList(); + // the binding of sql_block_rule name, multiple are separated by ',' + private String bindSqlBlockRules; + + @Deprecated + private byte[] password; + @Deprecated + private boolean isAdmin = false; + @Deprecated + private boolean isSuperuser = false; + @Deprecated + private Map dbPrivMap = Maps.newHashMap(); + static { ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_MAX_USER_CONNECTIONS + "$", Pattern.CASE_INSENSITIVE)); ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_RESOURCE + ".", Pattern.CASE_INSENSITIVE)); @@ -121,6 +134,15 @@ public long getMaxQueryInstances() { return commonProperties.getMaxQueryInstances();// maxQueryInstances; } + public String getBindSqlBlockRules() { + return bindSqlBlockRules; + } + + @Deprecated + public byte[] getPassword() { + return password; + } + public WhiteList getWhiteList() { return whiteList; } @@ -227,6 +249,12 @@ public void update(List> properties) throws DdlException { } catch (NumberFormatException e) { throw new DdlException(PROP_MAX_QUERY_INSTANCES + " is not number"); } + } else if (keyArr[0].equalsIgnoreCase(PROP_BIND_SQL_BLOCK_RULES)) { + // set property "bind_sql_block_rules" = "test_rule1,test_rule2" + if (keyArr.length != 1) { + throw new DdlException(PROP_BIND_SQL_BLOCK_RULES + " format error"); + } + bindSqlBlockRules = value; } else { throw new DdlException("Unknown user property(" + key + ")"); } @@ -381,6 +409,9 @@ public List> fetchProperty() { result.add(Lists.newArrayList("resolved IPs", Joiner.on(";").join(ips))); } + // bind_sql_block_rules + result.add(Lists.newArrayList(PROP_BIND_SQL_BLOCK_RULES, bindSqlBlockRules)); + // sort Collections.sort(result, new Comparator>() { @Override @@ -426,6 +457,8 @@ public void write(DataOutput out) throws IOException { // common properties commonProperties.write(out); + + Text.writeString(out, bindSqlBlockRules); } public void readFields(DataInput in) throws IOException { @@ -514,5 +547,9 @@ public void readFields(DataInput in) throws IOException { if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_100) { this.commonProperties = CommonUserProperties.read(in); } + + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_103) { + bindSqlBlockRules = Text.readString(in); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index 5f9cde8050ee77..cac61504f5fcda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -220,6 +220,14 @@ public void addUserPrivEntriesByResolvedIPs(Map> resolvedIPs } } + public String getBindSqlBlockRules(String qualifiedUser) { + UserProperty existProperty = propertyMap.get(qualifiedUser); + if (existProperty == null) { + return null; + } + return existProperty.getBindSqlBlockRules(); + } + public UserProperty getUserProperty(String qualifiedUserName) { return propertyMap.get(qualifiedUserName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 967f28431f57fa..b03d234156479b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -50,8 +50,8 @@ import org.apache.doris.thrift.TMasterOpResult; import org.apache.doris.thrift.TUniqueId; -import com.google.common.collect.Lists; import com.google.common.base.Strings; +import com.google.common.collect.Lists; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; @@ -183,6 +183,13 @@ private void handleQuery() { ctx.getState().setError("Unsupported character set(UTF-8)"); return; } + try { + Catalog.getCurrentCatalog().getSqlBlockRuleMgr().matchSql(originStmt, ctx.getQualifiedUser()); + } catch (AnalysisException e) { + LOG.error(e.getMessage()); + ctx.getState().setError(e.getMessage()); + return; + } ctx.getAuditEventBuilder().reset(); ctx.getAuditEventBuilder() .setTimestamp(System.currentTimeMillis()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9efa175866d441..04e481cb60f5ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -324,9 +324,6 @@ public void execute(TUniqueId queryId) throws Exception { } if (parsedStmt instanceof QueryStmt) { - QueryStmt queryStmt = (QueryStmt) this.parsedStmt; - // match sql - matchSql(queryStmt); context.getState().setIsQuery(true); MetricRepo.COUNTER_QUERY_BEGIN.increase(1L); int retryTime = Config.max_query_retry_time; @@ -1464,12 +1461,5 @@ private List exprToType(List exprs) { return exprs.stream().map(e -> e.getType().getPrimitiveType()).collect(Collectors.toList()); } - private void matchSql(QueryStmt queryStmt) throws AnalysisException { - String sql = queryStmt.getOrigStmt().originStmt; - String user = context.getUserIdentity().getUser(); - Catalog.getCurrentCatalog().getSqlBlockRuleMgr().matchSql(sql, user); - } - - } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java index 1183fb8afd2eca..e4a6beaa8d52d5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java @@ -54,7 +54,6 @@ public void setUp() { public void testNormal() throws UserException { Map properties = new HashMap<>(); properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); - properties.put(CreateSqlBlockRuleStmt.USER_PROPERTY, SqlBlockRule.DEFAULT_USER); properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); stmt.analyze(analyzer); diff --git a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java index df769f92651679..cd796ea0affc1c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java @@ -50,7 +50,7 @@ public void setUp() { @Test(expected = AnalysisException.class) public void testRegexMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; - SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", "default", ".* join .*", null, true); + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", ".* join .*", null, true, true); Pattern sqlPattern = Pattern.compile(sqlRule.getSql()); SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlPattern); } @@ -60,7 +60,7 @@ public void testHashMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; String hashSql = DigestUtils.md5Hex(sql); System.out.println(hashSql); - SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", "default", null, hashSql, true); + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, hashSql, true, true); SqlBlockRuleMgr.matchSql(sqlRule, sql, null); } @@ -69,7 +69,6 @@ public void testNormalCreate(@Mocked ConnectContext connectContext, @Mocked Cata SqlBlockRuleMgr sqlBlockRuleMgr = new SqlBlockRuleMgr(); Map properties = new HashMap<>(); properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); - properties.put(CreateSqlBlockRuleStmt.USER_PROPERTY, SqlBlockRule.DEFAULT_USER); properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); stmt.analyze(analyzer); From 1c2afcc4add49add7f3a8b0fea47908815ab4253 Mon Sep 17 00:00:00 2001 From: stalary Date: Thu, 29 Jul 2021 00:21:52 +0800 Subject: [PATCH 04/14] MOD: mod some code --- .../block-rule/sql-block.md | 2 +- .../block-rule/sql-block.md | 2 +- .../doris/analysis/AlterSqlBlockRuleStmt.java | 13 ++++ .../analysis/CreateSqlBlockRuleStmt.java | 22 +++++- .../doris/analysis/DropSqlBlockRuleStmt.java | 9 +++ .../doris/analysis/ShowSqlBlockRuleStmt.java | 27 +++++++ .../apache/doris/blockrule/SqlBlockRule.java | 6 ++ .../doris/blockrule/SqlBlockRuleMgr.java | 76 ++++++++----------- .../java/org/apache/doris/common/Config.java | 11 +-- .../apache/doris/journal/JournalEntity.java | 3 +- .../doris/mysql/privilege/PaloAuth.java | 10 +-- .../doris/mysql/privilege/UserProperty.java | 22 +++--- .../mysql/privilege/UserPropertyMgr.java | 4 +- .../persist/DropSqlBlockRuleOperationLog.java | 55 ++++++++++++++ .../org/apache/doris/persist/EditLog.java | 8 +- .../org/apache/doris/qe/ConnectContext.java | 10 +++ .../org/apache/doris/qe/ConnectProcessor.java | 10 ++- .../org/apache/doris/qe/ShowExecutor.java | 2 +- .../doris/blockrule/SqlBlockRuleMgrTest.java | 11 +-- 19 files changed, 212 insertions(+), 91 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/DropSqlBlockRuleOperationLog.java diff --git a/docs/en/administrator-guide/block-rule/sql-block.md b/docs/en/administrator-guide/block-rule/sql-block.md index 091181426763cf..8187745c86b0ad 100644 --- a/docs/en/administrator-guide/block-rule/sql-block.md +++ b/docs/en/administrator-guide/block-rule/sql-block.md @@ -55,5 +55,5 @@ DROP SQL_BLOCK_RULE test_rule1,test_rule2 ## User bind rules If global=false is configured, the rules binding for the specified user needs to be configured, with multiple rules separated by ', ' ``` -SET PROPERTY FOR 'jack' 'bind_sql_block_rules' = 'test_rule1,test_rule2' +SET PROPERTY FOR 'jack' 'sql_block_rules' = 'test_rule1,test_rule2' ``` diff --git a/docs/zh-CN/administrator-guide/block-rule/sql-block.md b/docs/zh-CN/administrator-guide/block-rule/sql-block.md index 48a113737b15e8..09cad23de71096 100644 --- a/docs/zh-CN/administrator-guide/block-rule/sql-block.md +++ b/docs/zh-CN/administrator-guide/block-rule/sql-block.md @@ -55,5 +55,5 @@ DROP SQL_BLOCK_RULE test_rule1,test_rule2 ## 用户规则绑定 如果配置global=false，则需要配置指定用户的规则绑定，多个规则使用`,`分隔 ``` -SET PROPERTY FOR 'jack' 'bind_sql_block_rules' = 'test_rule1,test_rule2' +SET PROPERTY FOR 'jack' 'sql_block_rules' = 'test_rule1,test_rule2' ``` \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java index 51ba67e9630f1e..6b7dfa004c1db7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java @@ -21,6 +21,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.PrintableMap; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -88,4 +89,16 @@ public Boolean getEnable() { public String getSqlHash() { return sqlHash; } + + @Override + public String toSql() { + // ALTER SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","enable"="true") + StringBuilder sb = new StringBuilder(); + sb.append("ALTER SQL_BLOCK_RULE ") + .append(ruleName) + .append(" \nPROPERTIES(\n") + .append(new PrintableMap<>(properties, " = ", true, true, true)) + .append(")"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java index 3e2f84a0149d50..efa599a27d2dc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java @@ -23,12 +23,15 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + import java.util.Map; import java.util.Optional; @@ -59,10 +62,10 @@ public class CreateSqlBlockRuleStmt extends DdlStmt { private String sqlHash; - // whether effective global + // whether effective global, default is false private boolean global; - // whether to use the rule + // whether to use the rule, default is true private boolean enable; private final Map properties; @@ -97,6 +100,10 @@ public void analyze(Analyzer analyzer) throws UserException { private void setProperties(Map properties) throws UserException { this.sql = properties.get(SQL_PROPERTY); this.sqlHash = properties.get(SQL_HASH_PROPERTY); + if ((StringUtils.isNotEmpty(sql) && StringUtils.isNotEmpty(sqlHash)) || (StringUtils.isEmpty(sql) && StringUtils.isEmpty(sqlHash))) { + throw new AnalysisException("Only one sql or sqlHash can be configured"); + } + this.global = Util.getBooleanPropertyOrDefault(properties.get(GLOBAL_PROPERTY), false, GLOBAL_PROPERTY + " should be a boolean"); this.enable = Util.getBooleanPropertyOrDefault(properties.get(ENABLE_PROPERTY), true, ENABLE_PROPERTY + " should be a boolean"); } @@ -131,4 +138,15 @@ public boolean isGlobal() { public boolean isEnable() { return enable; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE SQL_BLOCK_RULE ") + .append(ruleName) + .append(" \nPROPERTIES(\n") + .append(new PrintableMap<>(properties, " = ", true, true, true)) + .append(")"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java index 2c95e70df603fb..e885a10aa41da1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java @@ -24,6 +24,8 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; +import org.apache.parquet.Strings; + import java.util.List; public class DropSqlBlockRuleStmt extends DdlStmt { @@ -46,4 +48,11 @@ public DropSqlBlockRuleStmt(List ruleNames) { public List getRuleNames() { return ruleNames; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("DROP SQL_BLOCK_RULE ").append(Strings.join(ruleNames, ",")); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java index edb3500d075656..a7e07be20b94e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java @@ -17,10 +17,18 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.commons.lang3.StringUtils; + /* Create sqlBlockRule statement @@ -48,8 +56,27 @@ public String getRuleName() { return ruleName; } + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + } + @Override public ShowResultSetMetaData getMetaData() { return META_DATA; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("SHOW SQL_BLOCK_RULE"); + if (StringUtils.isNotEmpty(ruleName)) { + sb.append(" FOR ").append(ruleName); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java index a11ffc3e3fb65e..ee95b73c722f01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java @@ -24,6 +24,7 @@ import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.io.DataInput; import java.io.DataOutput; @@ -37,17 +38,22 @@ public class SqlBlockRule implements Writable { public static final String DEFAULT_USER = "default"; // the rule name, cluster unique + @SerializedName(value = "name") private String name; + @SerializedName(value = "sql") private String sql; // sql md5 + @SerializedName(value = "sqlHash") private String sqlHash; // whether effective global + @SerializedName(value = "global") private Boolean global; // whether to use the rule + @SerializedName(value = "enable") private Boolean enable; public SqlBlockRule(String name) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index 94179442d11b79..46be8178bd914a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -24,19 +24,17 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -55,8 +53,10 @@ public class SqlBlockRuleMgr implements Writable { private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + @SerializedName(value = "nameToSqlBlockRuleMap") private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); + @SerializedName(value = "sqlPatternMap") private Map sqlPatternMap = Maps.newConcurrentMap(); private void writeLock() { @@ -71,11 +71,8 @@ public boolean existRule(String name) { return nameToSqlBlockRuleMap.containsKey(name); } - public List get(ShowSqlBlockRuleStmt stmt) throws AnalysisException { + public List getSqlBlockRule(ShowSqlBlockRuleStmt stmt) throws AnalysisException { String ruleName = stmt.getRuleName(); - if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); - } if (StringUtils.isNotEmpty(ruleName)) { if (nameToSqlBlockRuleMap.containsKey(ruleName)) { SqlBlockRule sqlBlockRule = nameToSqlBlockRuleMap.get(ruleName); @@ -159,75 +156,66 @@ public void dropSqlBlockRule(DropSqlBlockRuleStmt stmt) throws DdlException { if (!existRule(ruleName)) { throw new DdlException("the sql block rule " + ruleName + " not exist"); } - SqlBlockRule sqlBlockRule = nameToSqlBlockRuleMap.get(ruleName); - if (sqlBlockRule == null) { - continue; - } - unprotectedDrop(sqlBlockRule); - Catalog.getCurrentCatalog().getEditLog().logDropSqlBlockRule(sqlBlockRule); } + unprotectedDrop(ruleNames); + Catalog.getCurrentCatalog().getEditLog().logDropSqlBlockRule(ruleNames); } finally { writeUnlock(); } } - public void replayDrop(SqlBlockRule sqlBlockRule) { - unprotectedDrop(sqlBlockRule); - LOG.info("replay drop sql block rule: {}", sqlBlockRule); + public void replayDrop(List ruleNames) { + unprotectedDrop(ruleNames); + LOG.info("replay drop sql block ruleNames: {}", ruleNames); } - public void unprotectedDrop(SqlBlockRule sqlBlockRule) { - nameToSqlBlockRuleMap.remove(sqlBlockRule.getName()); - // todo: remove UserProperty + public void unprotectedDrop(List ruleNames) { + ruleNames.forEach(name -> nameToSqlBlockRuleMap.remove(name)); } - public void matchSql(String sql, String user) throws AnalysisException { + public void matchSql(String sql, String sqlHash, String user) throws AnalysisException { // match global rule List globalRules = nameToSqlBlockRuleMap.values().stream().filter(SqlBlockRule::getGlobal).collect(Collectors.toList()); for (SqlBlockRule rule : globalRules) { Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); - matchSql(rule, sql, sqlPattern); + matchSql(rule, sql, sqlHash, sqlPattern); } // match user rule - String binSqlBlockRules = Catalog.getCurrentCatalog().getAuth().getBindSqlBlockRules(user); - if (StringUtils.isNotEmpty(binSqlBlockRules)) { - String[] split = binSqlBlockRules.split(","); + String bindSqlBlockRules = Catalog.getCurrentCatalog().getAuth().getSqlBlockRules(user); + if (StringUtils.isNotEmpty(bindSqlBlockRules)) { + String[] split = bindSqlBlockRules.split(","); for (String ruleName : split) { SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName); Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); - matchSql(rule, sql, sqlPattern); + matchSql(rule, sql, sqlHash, sqlPattern); } } } @VisibleForTesting - public static void matchSql(SqlBlockRule rule, String sql, Pattern sqlPattern) throws AnalysisException { - if (rule.getEnable() != null && rule.getEnable()) { - String sqlHash = rule.getSqlHash(); - if (sqlHash != null && sqlHash.equals(DigestUtils.md5Hex(sql))) { + public static void matchSql(SqlBlockRule rule, String sql, String sqlHash, Pattern sqlPattern) throws AnalysisException { + if (rule.getEnable()) { + if (StringUtils.isNotEmpty(rule.getSql())) { + if (sqlPattern != null && sqlPattern.matcher(sql).find()) { + MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); + throw new AnalysisException("sql match regex sql block rule: " + rule.getName()); + } + } else if (StringUtils.isNotEmpty(rule.getSqlHash()) && rule.getSqlHash().equals(sqlHash)) { MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); throw new AnalysisException("sql match hash sql block rule: " + rule.getName()); } - if (sqlPattern != null && sqlPattern.matcher(sql).find()) { - MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); - throw new AnalysisException("sql match regex sql block rule: " + rule.getName()); - } } } @Override public void write(DataOutput out) throws IOException { - out.writeInt(nameToSqlBlockRuleMap.size()); - for (SqlBlockRule sqlBlockRule : nameToSqlBlockRuleMap.values()) { - sqlBlockRule.write(out); - } + Text.writeString(out, GsonUtils.GSON.toJson(this)); } public void readFields(DataInput in) throws IOException { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - SqlBlockRule read = SqlBlockRule.read(in); - unprotectedAdd(read); - } + String json = Text.readString(in); + SqlBlockRuleMgr mgr = GsonUtils.GSON.fromJson(json, SqlBlockRuleMgr.class); + this.nameToSqlBlockRuleMap = mgr.nameToSqlBlockRuleMap; + this.sqlPatternMap = mgr.sqlPatternMap; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 76a8d9a0946b9d..ed2751bd3feaff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1474,16 +1474,7 @@ public class Config extends ConfigBase { */ @ConfField(mutable = false, masterOnly = true) public static int partition_in_memory_update_interval_secs = 300; -<<<<<<< HEAD - + @ConfField(masterOnly = true) public static boolean enable_concurrent_update = false; -======= - - /** - * Whether to enabel block sql - */ - @ConfField(mutable = true, masterOnly = false) - public static boolean enable_sql_block = false; ->>>>>>> 2f8cca53c (ADD: support sql block rule) } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index e9469ffd40ad76..4bdb0549adaf1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -67,6 +67,7 @@ import org.apache.doris.persist.DropLinkDbAndUpdateDbInfo; import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.persist.DropResourceOperationLog; +import org.apache.doris.persist.DropSqlBlockRuleOperationLog; import org.apache.doris.persist.GlobalVarPersistInfo; import org.apache.doris.persist.HbPackage; import org.apache.doris.persist.LdapInfo; @@ -650,7 +651,7 @@ public void readFields(DataInput in) throws IOException { break; } case OperationType.OP_DROP_SQL_BLOCK_RULE: { - data = SqlBlockRule.read(in); + data = DropSqlBlockRuleOperationLog.read(in); isRead = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java index 11aaf7dd9bef68..e806f5fc1ac88c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java @@ -50,14 +50,14 @@ import org.apache.doris.thrift.TFetchResourceResult; import org.apache.doris.thrift.TPrivilegeStatus; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -1134,10 +1134,10 @@ public long getMaxQueryInstances(String qualifiedUser) { } } - public String getBindSqlBlockRules(String qualifiedUser) { + public String getSqlBlockRules(String qualifiedUser) { readLock(); try { - return propertyMgr.getBindSqlBlockRules(qualifiedUser); + return propertyMgr.getSqlBlockRules(qualifiedUser); } finally { readUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index 49784c1501d6c8..32471ea60d4b4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -66,7 +66,7 @@ public class UserProperty implements Writable { private static final String PROP_QUOTA = "quota"; private static final String PROP_DEFAULT_LOAD_CLUSTER = "default_load_cluster"; private static final String PROP_LOAD_CLUSTER = "load_cluster"; - private static final String PROP_BIND_SQL_BLOCK_RULES = "bind_sql_block_rules"; + private static final String PROP_SQL_BLOCK_RULES = "sql_block_rules"; // for system user public static final Set ADVANCED_PROPERTIES = Sets.newHashSet(); @@ -91,7 +91,7 @@ public class UserProperty implements Writable { private WhiteList whiteList = new WhiteList(); // the binding of sql_block_rule name, multiple are separated by ',' - private String bindSqlBlockRules; + private String sqlBlockRules; @Deprecated private byte[] password; @@ -134,8 +134,8 @@ public long getMaxQueryInstances() { return commonProperties.getMaxQueryInstances();// maxQueryInstances; } - public String getBindSqlBlockRules() { - return bindSqlBlockRules; + public String getSqlBlockRules() { + return sqlBlockRules; } @Deprecated @@ -249,12 +249,12 @@ public void update(List> properties) throws DdlException { } catch (NumberFormatException e) { throw new DdlException(PROP_MAX_QUERY_INSTANCES + " is not number"); } - } else if (keyArr[0].equalsIgnoreCase(PROP_BIND_SQL_BLOCK_RULES)) { - // set property "bind_sql_block_rules" = "test_rule1,test_rule2" + } else if (keyArr[0].equalsIgnoreCase(PROP_SQL_BLOCK_RULES)) { + // set property "sql_block_rules" = "test_rule1,test_rule2" if (keyArr.length != 1) { - throw new DdlException(PROP_BIND_SQL_BLOCK_RULES + " format error"); + throw new DdlException(PROP_SQL_BLOCK_RULES + " format error"); } - bindSqlBlockRules = value; + sqlBlockRules = value; } else { throw new DdlException("Unknown user property(" + key + ")"); } @@ -410,7 +410,7 @@ public List> fetchProperty() { } // bind_sql_block_rules - result.add(Lists.newArrayList(PROP_BIND_SQL_BLOCK_RULES, bindSqlBlockRules)); + result.add(Lists.newArrayList(PROP_SQL_BLOCK_RULES, sqlBlockRules)); // sort Collections.sort(result, new Comparator>() { @@ -458,7 +458,7 @@ public void write(DataOutput out) throws IOException { // common properties commonProperties.write(out); - Text.writeString(out, bindSqlBlockRules); + Text.writeString(out, sqlBlockRules); } public void readFields(DataInput in) throws IOException { @@ -549,7 +549,7 @@ public void readFields(DataInput in) throws IOException { } if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_103) { - bindSqlBlockRules = Text.readString(in); + sqlBlockRules = Text.readString(in); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index cac61504f5fcda..6594ede7cb33a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -220,12 +220,12 @@ public void addUserPrivEntriesByResolvedIPs(Map> resolvedIPs } } - public String getBindSqlBlockRules(String qualifiedUser) { + public String getSqlBlockRules(String qualifiedUser) { UserProperty existProperty = propertyMap.get(qualifiedUser); if (existProperty == null) { return null; } - return existProperty.getBindSqlBlockRules(); + return existProperty.getSqlBlockRules(); } public UserProperty getUserProperty(String qualifiedUserName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropSqlBlockRuleOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropSqlBlockRuleOperationLog.java new file mode 100644 index 00000000000000..f3d64a8e6287b2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropSqlBlockRuleOperationLog.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +/** + * For user sql_block_rule drop + */ +public class DropSqlBlockRuleOperationLog implements Writable { + + @SerializedName(value = "ruleNames") + private List ruleNames; + + public DropSqlBlockRuleOperationLog(List ruleNames) { + this.ruleNames = ruleNames; + } + + public List getRuleNames() { + return ruleNames; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static DropSqlBlockRuleOperationLog read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), DropSqlBlockRuleOperationLog.class); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index a1a538dae54558..c0f562b189cbb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -855,8 +855,8 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { break; } case OperationType.OP_DROP_SQL_BLOCK_RULE: { - SqlBlockRule rule = (SqlBlockRule) journal.getData(); - catalog.getSqlBlockRuleMgr().replayDrop(rule); + DropSqlBlockRuleOperationLog log = (DropSqlBlockRuleOperationLog) journal.getData(); + catalog.getSqlBlockRuleMgr().replayDrop(log.getRuleNames()); break; } default: { @@ -1471,7 +1471,7 @@ public void logAlterSqlBlockRule(SqlBlockRule rule) { logEdit(OperationType.OP_ALTER_SQL_BLOCK_RULE, rule); } - public void logDropSqlBlockRule(SqlBlockRule rule) { - logEdit(OperationType.OP_DROP_SQL_BLOCK_RULE, rule); + public void logDropSqlBlockRule(List ruleNames) { + logEdit(OperationType.OP_DROP_SQL_BLOCK_RULE, new DropSqlBlockRuleOperationLog(ruleNames)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 16fc2d7681be6b..a39939c015ee74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -116,6 +116,8 @@ public class ConnectContext { // If set to true, the nondeterministic function will not be rewrote to constant. private boolean notEvalNondeterministicFunction = false; + private String sqlHash; + public static ConnectContext get() { return threadLocalInfo.get(); } @@ -419,6 +421,14 @@ public void setCluster(String clusterName) { this.clusterName = clusterName; } + public String getSqlHash() { + return sqlHash; + } + + public void setSqlHash(String sqlHash) { + this.sqlHash = sqlHash; + } + // kill operation with no protect. public void kill(boolean killConnection) { LOG.warn("kill timeout query, {}, kill connection: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index b03d234156479b..62e82c7eb35b71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -158,7 +158,6 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue } sql = origStmt; } - ctx.getAuditEventBuilder().setSqlHash(DigestUtils.md5Hex(sql)); // We put origin query stmt at the end of audit log, for parsing the log more convenient. ctx.getAuditEventBuilder().setStmt(sql); Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); @@ -183,10 +182,12 @@ private void handleQuery() { ctx.getState().setError("Unsupported character set(UTF-8)"); return; } + String sqlHash = DigestUtils.md5Hex(originStmt); + ctx.setSqlHash(sqlHash); try { - Catalog.getCurrentCatalog().getSqlBlockRuleMgr().matchSql(originStmt, ctx.getQualifiedUser()); + Catalog.getCurrentCatalog().getSqlBlockRuleMgr().matchSql(originStmt, sqlHash, ctx.getQualifiedUser()); } catch (AnalysisException e) { - LOG.error(e.getMessage()); + LOG.warn(e.getMessage()); ctx.getState().setError(e.getMessage()); return; } @@ -195,7 +196,8 @@ private void handleQuery() { .setTimestamp(System.currentTimeMillis()) .setClientIp(ctx.getMysqlChannel().getRemoteHostPortString()) .setUser(ctx.getQualifiedUser()) - .setDb(ctx.getDatabase()); + .setDb(ctx.getDatabase()) + .setSqlHash(ctx.getSqlHash()); // execute this query. StatementBase parsedStmt = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8aae2e18eb2d57..00c66ff3fa2f35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2096,7 +2096,7 @@ private void handleAdminShowDataSkew() throws AnalysisException { public void handleShowSqlBlockRule() throws AnalysisException { ShowSqlBlockRuleStmt showStmt = (ShowSqlBlockRuleStmt) stmt; List> rows = Lists.newArrayList(); - List sqlBlockRules = Catalog.getCurrentCatalog().getSqlBlockRuleMgr().get(showStmt); + List sqlBlockRules = Catalog.getCurrentCatalog().getSqlBlockRuleMgr().getSqlBlockRule(showStmt); sqlBlockRules.forEach(rule -> rows.add(rule.getShowInfo())); resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java index cd796ea0affc1c..d53b2eb6489466 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java @@ -50,18 +50,19 @@ public void setUp() { @Test(expected = AnalysisException.class) public void testRegexMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; + String sqlHash = DigestUtils.md5Hex(sql); SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", ".* join .*", null, true, true); Pattern sqlPattern = Pattern.compile(sqlRule.getSql()); - SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlPattern); + SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlHash, sqlPattern); } @Test(expected = AnalysisException.class) public void testHashMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; - String hashSql = DigestUtils.md5Hex(sql); - System.out.println(hashSql); - SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, hashSql, true, true); - SqlBlockRuleMgr.matchSql(sqlRule, sql, null); + String sqlHash = DigestUtils.md5Hex(sql); + System.out.println(sqlHash); + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, sqlHash, true, true); + SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlHash, null); } @Test From c68bddf61debf79f0a41180a59f7048c224123ef Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 8 Aug 2021 19:47:46 +0800 Subject: [PATCH 05/14] FIX: fix some bug --- fe/fe-core/src/main/cup/sql_parser.cup | 3 +- .../doris/analysis/ShowSqlBlockRuleStmt.java | 3 +- .../apache/doris/blockrule/SqlBlockRule.java | 2 +- .../doris/blockrule/SqlBlockRuleMgr.java | 29 ++++++++++--------- .../org/apache/doris/catalog/Catalog.java | 2 +- .../doris/mysql/privilege/UserProperty.java | 9 ++++-- .../org/apache/doris/qe/ShowExecutor.java | 1 - .../doris/blockrule/SqlBlockRuleMgrTest.java | 11 ++++--- 8 files changed, 34 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 4e31d4047539df..232d9fb8642a91 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -827,7 +827,7 @@ alter_stmt ::= {: RESULT = new AlterRoutineLoadStmt(jobLabel, jobProperties, datasourceProperties); :} - | KW_ALTER KW_SQL_BLOCK_RULE STRING_LITERAL:ruleName + | KW_ALTER KW_SQL_BLOCK_RULE ident:ruleName opt_properties:properties {: RESULT = new AlterSqlBlockRuleStmt(ruleName, properties); @@ -1259,6 +1259,7 @@ create_stmt ::= {: RESULT = new CreateDataSyncJobStmt(jobName, db, channelDescList, binlog, properties); :} + /* sql_block_rule */ | KW_CREATE KW_SQL_BLOCK_RULE ident:ruleName opt_properties:properties {: RESULT = new CreateSqlBlockRuleStmt(ruleName, properties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java index a7e07be20b94e1..c93237fa02a05f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java @@ -41,8 +41,9 @@ public class ShowSqlBlockRuleStmt extends ShowStmt { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Name", ScalarType.createVarchar(50))) - .addColumn(new Column("User", ScalarType.createVarchar(30))) .addColumn(new Column("Sql", ScalarType.createVarchar(65535))) + .addColumn(new Column("SqlHash", ScalarType.createVarchar(65535))) + .addColumn(new Column("Global", ScalarType.createVarchar(4))) .addColumn(new Column("Enable", ScalarType.createVarchar(4))) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java index ee95b73c722f01..fdb4adb3c25597 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java @@ -113,7 +113,7 @@ public void setEnable(Boolean enable) { } public List getShowInfo() { - return Lists.newArrayList(this.name, this.sql, String.valueOf(this.enable)); + return Lists.newArrayList(this.name, this.sql, this.sqlHash, String.valueOf(this.global), String.valueOf(this.enable)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index 46be8178bd914a..5520cb94cc6bed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -30,7 +30,6 @@ import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.gson.GsonUtils; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -173,38 +172,40 @@ public void unprotectedDrop(List ruleNames) { ruleNames.forEach(name -> nameToSqlBlockRuleMap.remove(name)); } - public void matchSql(String sql, String sqlHash, String user) throws AnalysisException { + public void matchSql(String originSql, String sqlHash, String user) throws AnalysisException { // match global rule List globalRules = nameToSqlBlockRuleMap.values().stream().filter(SqlBlockRule::getGlobal).collect(Collectors.toList()); for (SqlBlockRule rule : globalRules) { - Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); - matchSql(rule, sql, sqlHash, sqlPattern); + matchSql(rule, originSql, sqlHash); } // match user rule String bindSqlBlockRules = Catalog.getCurrentCatalog().getAuth().getSqlBlockRules(user); if (StringUtils.isNotEmpty(bindSqlBlockRules)) { String[] split = bindSqlBlockRules.split(","); for (String ruleName : split) { - SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName); - Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); - matchSql(rule, sql, sqlHash, sqlPattern); + SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName.trim()); + if (rule == null) { + continue; + } + matchSql(rule, originSql, sqlHash); } } } - @VisibleForTesting - public static void matchSql(SqlBlockRule rule, String sql, String sqlHash, Pattern sqlPattern) throws AnalysisException { + public void matchSql(SqlBlockRule rule, String originSql, String sqlHash) throws AnalysisException { if (rule.getEnable()) { - if (StringUtils.isNotEmpty(rule.getSql())) { - if (sqlPattern != null && sqlPattern.matcher(sql).find()) { + if (StringUtils.isNotEmpty(rule.getSqlHash()) && rule.getSqlHash().equals(sqlHash)) { + MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); + throw new AnalysisException("sql match hash sql block rule: " + rule.getName()); + } else if (StringUtils.isNotEmpty(rule.getSql())) { + Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); + if (sqlPattern.matcher(originSql).find()) { MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); throw new AnalysisException("sql match regex sql block rule: " + rule.getName()); } - } else if (StringUtils.isNotEmpty(rule.getSqlHash()) && rule.getSqlHash().equals(sqlHash)) { - MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); - throw new AnalysisException("sql match hash sql block rule: " + rule.getName()); } } + } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 8287cb99307212..5820fd3952390d 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1905,7 +1905,7 @@ public long loadSmallFiles(DataInputStream in, long checksum) throws IOException } public long loadSqlBlockRule(DataInputStream in, long checksum) throws IOException { - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_100) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_103) { sqlBlockRuleMgr.readFields(in); } LOG.info("finished replay sqlBlockRule from image"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index 32471ea60d4b4c..8470e003fa6000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -91,7 +91,7 @@ public class UserProperty implements Writable { private WhiteList whiteList = new WhiteList(); // the binding of sql_block_rule name, multiple are separated by ',' - private String sqlBlockRules; + private String sqlBlockRules = null; @Deprecated private byte[] password; @@ -108,6 +108,7 @@ public class UserProperty implements Writable { ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_LOAD_CLUSTER + "." + DppConfig.CLUSTER_NAME_REGEX + "." + DppConfig.PRIORITY + "$", Pattern.CASE_INSENSITIVE)); ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_MAX_QUERY_INSTANCES + "$", Pattern.CASE_INSENSITIVE)); + ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_SQL_BLOCK_RULES + "$", Pattern.CASE_INSENSITIVE)); COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUOTA + ".", Pattern.CASE_INSENSITIVE)); COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE)); @@ -458,7 +459,9 @@ public void write(DataOutput out) throws IOException { // common properties commonProperties.write(out); - Text.writeString(out, sqlBlockRules); + if (StringUtils.isNotEmpty(sqlBlockRules)) { + Text.writeString(out, sqlBlockRules); + } } public void readFields(DataInput in) throws IOException { @@ -466,7 +469,7 @@ public void readFields(DataInput in) throws IOException { // consume the flag of empty user name in.readBoolean(); } - + // user name if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_30) { qualifiedUser = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, Text.readString(in)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 00c66ff3fa2f35..df776fbaa65038 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2081,7 +2081,6 @@ private void handleShowCreateRoutineLoad() throws AnalysisException { resultSet = new ShowResultSet(showCreateRoutineLoadStmt.getMetaData(), rows); } -<<<<<<< HEAD private void handleAdminShowDataSkew() throws AnalysisException { AdminShowDataSkewStmt showStmt = (AdminShowDataSkewStmt) stmt; List> results; diff --git a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java index d53b2eb6489466..0a7b054e0e98f8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java @@ -32,7 +32,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.regex.Pattern; import mockit.Mocked; @@ -44,6 +43,7 @@ public class SqlBlockRuleMgrTest { public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); MetricRepo.init(); + } @@ -52,8 +52,9 @@ public void testRegexMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; String sqlHash = DigestUtils.md5Hex(sql); SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", ".* join .*", null, true, true); - Pattern sqlPattern = Pattern.compile(sqlRule.getSql()); - SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlHash, sqlPattern); + SqlBlockRuleMgr mgr = new SqlBlockRuleMgr(); + mgr.replayCreate(sqlRule); + mgr.matchSql(sqlRule, sql, sqlHash); } @Test(expected = AnalysisException.class) @@ -62,7 +63,9 @@ public void testHashMatchSql() throws AnalysisException { String sqlHash = DigestUtils.md5Hex(sql); System.out.println(sqlHash); SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, sqlHash, true, true); - SqlBlockRuleMgr.matchSql(sqlRule, sql, sqlHash, null); + SqlBlockRuleMgr mgr = new SqlBlockRuleMgr(); + mgr.replayCreate(sqlRule); + mgr.matchSql(sqlRule, sql, sqlHash); } @Test From d58e18c6be0ac5476eba982a91e31a1e7d0e30fa Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 8 Aug 2021 20:36:46 +0800 Subject: [PATCH 06/14] MOD: sidebar --- docs/.vuepress/sidebar/en.js | 8 ++++++++ docs/.vuepress/sidebar/zh-CN.js | 9 ++++++++- docs/en/administrator-guide/block-rule/sql-block.md | 2 +- docs/zh-CN/administrator-guide/block-rule/sql-block.md | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index cf409d5fb0a26e..d9cd5b5240ecfb 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -182,6 +182,14 @@ module.exports = [ ], sidebarDepth: 1, }, + { + title: "Block Rule", + directoryPath: "block-rule/", + children: [ + "sql-block", + ], + sidebarDepth: 1, + }, "backup-restore", "broker", "colocation-join", diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index ed0ba4f1d2cb04..c4fd9b7035e50e 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -181,6 +181,14 @@ module.exports = [ ], sidebarDepth: 1, }, + { + title: "拦截规则", + directoryPath: "block-rule/", + children: [ + "sql-block", + ], + sidebarDepth: 1, + }, "backup-restore", "broker", "colocation-join", @@ -200,7 +208,6 @@ module.exports = [ "time-zone", "variables", "update", - "sql-block", ], sidebarDepth: 1, }, diff --git a/docs/en/administrator-guide/block-rule/sql-block.md b/docs/en/administrator-guide/block-rule/sql-block.md index 8187745c86b0ad..ddbd1e5e5c568a 100644 --- a/docs/en/administrator-guide/block-rule/sql-block.md +++ b/docs/en/administrator-guide/block-rule/sql-block.md @@ -55,5 +55,5 @@ DROP SQL_BLOCK_RULE test_rule1,test_rule2 ## User bind rules If global=false is configured, the rules binding for the specified user needs to be configured, with multiple rules separated by ', ' ``` -SET PROPERTY FOR 'jack' 'sql_block_rules' = 'test_rule1,test_rule2' +SET PROPERTY [FOR 'jack'] 'sql_block_rules' = 'test_rule1,test_rule2' ``` diff --git a/docs/zh-CN/administrator-guide/block-rule/sql-block.md b/docs/zh-CN/administrator-guide/block-rule/sql-block.md index 09cad23de71096..bfaa16767dcbb4 100644 --- a/docs/zh-CN/administrator-guide/block-rule/sql-block.md +++ b/docs/zh-CN/administrator-guide/block-rule/sql-block.md @@ -55,5 +55,5 @@ DROP SQL_BLOCK_RULE test_rule1,test_rule2 ## 用户规则绑定 如果配置global=false，则需要配置指定用户的规则绑定，多个规则使用`,`分隔 ``` -SET PROPERTY FOR 'jack' 'sql_block_rules' = 'test_rule1,test_rule2' +SET PROPERTY [FOR 'jack'] 'sql_block_rules' = 'test_rule1,test_rule2' ``` \ No newline at end of file From 919cf7aa2f5859998f205dbf418d3df1ad08855a Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 8 Aug 2021 20:48:03 +0800 Subject: [PATCH 07/14] MOD: sidebar --- docs/.vuepress/sidebar/en.js | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index d9cd5b5240ecfb..cf8012e7b2d218 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -207,7 +207,6 @@ module.exports = [ "time-zone", "variables", "update", - "sql-block", ], sidebarDepth: 1, }, From 961e5e18e4710975373918563be1b0e781298225 Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 8 Aug 2021 23:24:20 +0800 Subject: [PATCH 08/14] MOD: mod some code --- .../apache/doris/blockrule/SqlBlockRule.java | 20 +++++++- .../doris/blockrule/SqlBlockRuleMgr.java | 39 +++++---------- .../org/apache/doris/catalog/Catalog.java | 2 +- .../mysql/privilege/CommonUserProperties.java | 10 ++++ .../doris/mysql/privilege/PaloAuth.java | 2 +- .../doris/mysql/privilege/UserProperty.java | 47 +++++++------------ .../mysql/privilege/UserPropertyMgr.java | 4 +- .../doris/catalog/UserPropertyTest.java | 10 ++++ 8 files changed, 71 insertions(+), 63 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java index fdb4adb3c25597..13bbfbe51afdb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java @@ -26,10 +26,13 @@ import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.List; +import java.util.regex.Pattern; public class SqlBlockRule implements Writable { @@ -56,6 +59,8 @@ public class SqlBlockRule implements Writable { @SerializedName(value = "enable") private Boolean enable; + private Pattern sqlPattern; + public SqlBlockRule(String name) { this.name = name; } @@ -66,6 +71,9 @@ public SqlBlockRule(String name, String sql, String sqlHash, Boolean global, Boo this.sqlHash = sqlHash; this.global = global; this.enable = enable; + if (StringUtils.isNotEmpty(sql)) { + this.sqlPattern = Pattern.compile(sql); + } } public static SqlBlockRule fromCreateStmt(CreateSqlBlockRuleStmt stmt) { @@ -84,6 +92,10 @@ public String getSql() { return sql; } + public Pattern getSqlPattern() { + return sqlPattern; + } + public String getSqlHash() { return sqlHash; } @@ -100,6 +112,10 @@ public void setSql(String sql) { this.sql = sql; } + public void setSqlPattern(Pattern sqlPattern) { + this.sqlPattern = sqlPattern; + } + public void setSqlHash(String sqlHash) { this.sqlHash = sqlHash; } @@ -123,6 +139,8 @@ public void write(DataOutput out) throws IOException { public static SqlBlockRule read(DataInput in) throws IOException { String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, SqlBlockRule.class); + SqlBlockRule sqlBlockRule = GsonUtils.GSON.fromJson(json, SqlBlockRule.class); + sqlBlockRule.setSqlPattern(Pattern.compile(sqlBlockRule.getSql())); + return sqlBlockRule; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index 5520cb94cc6bed..fa454a6ef81ab3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.regex.Pattern; import java.util.stream.Collectors; public class SqlBlockRuleMgr implements Writable { @@ -55,9 +54,6 @@ public class SqlBlockRuleMgr implements Writable { @SerializedName(value = "nameToSqlBlockRuleMap") private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); - @SerializedName(value = "sqlPatternMap") - private Map sqlPatternMap = Maps.newConcurrentMap(); - private void writeLock() { lock.writeLock().lock(); } @@ -141,10 +137,6 @@ public void unprotectedUpdate(SqlBlockRule sqlBlockRule) { public void unprotectedAdd(SqlBlockRule sqlBlockRule) { nameToSqlBlockRuleMap.put(sqlBlockRule.getName(), sqlBlockRule); - String sql = sqlBlockRule.getSql(); - if (StringUtils.isNotEmpty(sql)) { - sqlPatternMap.put(sql, Pattern.compile(sql)); - } } public void dropSqlBlockRule(DropSqlBlockRuleStmt stmt) throws DdlException { @@ -179,16 +171,13 @@ public void matchSql(String originSql, String sqlHash, String user) throws Analy matchSql(rule, originSql, sqlHash); } // match user rule - String bindSqlBlockRules = Catalog.getCurrentCatalog().getAuth().getSqlBlockRules(user); - if (StringUtils.isNotEmpty(bindSqlBlockRules)) { - String[] split = bindSqlBlockRules.split(","); - for (String ruleName : split) { - SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName.trim()); - if (rule == null) { - continue; - } - matchSql(rule, originSql, sqlHash); + String[] bindSqlBlockRules = Catalog.getCurrentCatalog().getAuth().getSqlBlockRules(user); + for (String ruleName : bindSqlBlockRules) { + SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName); + if (rule == null) { + continue; } + matchSql(rule, originSql, sqlHash); } } @@ -197,15 +186,11 @@ public void matchSql(SqlBlockRule rule, String originSql, String sqlHash) throws if (StringUtils.isNotEmpty(rule.getSqlHash()) && rule.getSqlHash().equals(sqlHash)) { MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); throw new AnalysisException("sql match hash sql block rule: " + rule.getName()); - } else if (StringUtils.isNotEmpty(rule.getSql())) { - Pattern sqlPattern = sqlPatternMap.get(rule.getSql()); - if (sqlPattern.matcher(originSql).find()) { - MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); - throw new AnalysisException("sql match regex sql block rule: " + rule.getName()); - } + } else if (StringUtils.isNotEmpty(rule.getSql()) && rule.getSqlPattern().matcher(originSql).find()) { + MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L); + throw new AnalysisException("sql match regex sql block rule: " + rule.getName()); } } - } @Override @@ -213,10 +198,8 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); } - public void readFields(DataInput in) throws IOException { + public static SqlBlockRuleMgr read(DataInput in) throws IOException { String json = Text.readString(in); - SqlBlockRuleMgr mgr = GsonUtils.GSON.fromJson(json, SqlBlockRuleMgr.class); - this.nameToSqlBlockRuleMap = mgr.nameToSqlBlockRuleMap; - this.sqlPatternMap = mgr.sqlPatternMap; + return GsonUtils.GSON.fromJson(json, SqlBlockRuleMgr.class); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 5820fd3952390d..2ecdd60d2ea0c6 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1906,7 +1906,7 @@ public long loadSmallFiles(DataInputStream in, long checksum) throws IOException public long loadSqlBlockRule(DataInputStream in, long checksum) throws IOException { if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_103) { - sqlBlockRuleMgr.readFields(in); + sqlBlockRuleMgr = SqlBlockRuleMgr.read(in); } LOG.info("finished replay sqlBlockRule from image"); return checksum; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index 0695f8a0455a84..605aee63a3f704 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -35,6 +35,8 @@ public class CommonUserProperties implements Writable { private long maxConn = 100; @SerializedName("maxQueryInstances") private long maxQueryInstances = -1; + @SerializedName("sqlBlockRules") + private String sqlBlockRules = ""; long getMaxConn() { return maxConn; @@ -44,6 +46,10 @@ long getMaxQueryInstances() { return maxQueryInstances; } + String getSqlBlockRules() { + return sqlBlockRules; + } + void setMaxConn(long maxConn) { this.maxConn = maxConn; } @@ -52,6 +58,10 @@ void setMaxQueryInstances(long maxQueryInstances) { this.maxQueryInstances = maxQueryInstances; } + void setSqlBlockRules(String sqlBlockRules) { + this.sqlBlockRules = sqlBlockRules; + } + public static CommonUserProperties read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, CommonUserProperties.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java index e806f5fc1ac88c..836ce39128ec1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java @@ -1134,7 +1134,7 @@ public long getMaxQueryInstances(String qualifiedUser) { } } - public String getSqlBlockRules(String qualifiedUser) { + public String[] getSqlBlockRules(String qualifiedUser) { readLock(); try { return propertyMgr.getSqlBlockRules(qualifiedUser); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index 8470e003fa6000..3ee118c83e6f50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -91,16 +91,7 @@ public class UserProperty implements Writable { private WhiteList whiteList = new WhiteList(); // the binding of sql_block_rule name, multiple are separated by ',' - private String sqlBlockRules = null; - - @Deprecated - private byte[] password; - @Deprecated - private boolean isAdmin = false; - @Deprecated - private boolean isSuperuser = false; - @Deprecated - private Map dbPrivMap = Maps.newHashMap(); + private String[] sqlBlockRulesSplit = {}; static { ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_MAX_USER_CONNECTIONS + "$", Pattern.CASE_INSENSITIVE)); @@ -135,13 +126,15 @@ public long getMaxQueryInstances() { return commonProperties.getMaxQueryInstances();// maxQueryInstances; } - public String getSqlBlockRules() { - return sqlBlockRules; - } - - @Deprecated - public byte[] getPassword() { - return password; + public String[] getSqlBlockRules() { + if (this.sqlBlockRulesSplit.length != 0) { + return this.sqlBlockRulesSplit; + } + String sqlBlockRules = commonProperties.getSqlBlockRules(); + if (StringUtils.isNotEmpty(sqlBlockRules)) { + this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); + } + return this.sqlBlockRulesSplit; } public WhiteList getWhiteList() { @@ -166,6 +159,7 @@ public void update(List> properties) throws DdlException { // copy long newMaxConn = this.commonProperties.getMaxConn(); long newMaxQueryInstances = this.commonProperties.getMaxQueryInstances(); + String sqlBlockRules = this.commonProperties.getSqlBlockRules(); UserResource newResource = resource.getCopiedUserResource(); String newDefaultLoadCluster = defaultLoadCluster; Map newDppConfigs = Maps.newHashMap(clusterToDppConfig); @@ -264,6 +258,8 @@ public void update(List> properties) throws DdlException { // set this.commonProperties.setMaxConn(newMaxConn); this.commonProperties.setMaxQueryInstances(newMaxQueryInstances); + this.commonProperties.setSqlBlockRules(sqlBlockRules); + this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); resource = newResource; if (newDppConfigs.containsKey(newDefaultLoadCluster)) { defaultLoadCluster = newDefaultLoadCluster; @@ -351,6 +347,9 @@ public List> fetchProperty() { // max query instance result.add(Lists.newArrayList(PROP_MAX_QUERY_INSTANCES, String.valueOf(commonProperties.getMaxQueryInstances()))); + // sql block rules + result.add(Lists.newArrayList(PROP_SQL_BLOCK_RULES, commonProperties.getSqlBlockRules())); + // resource ResourceGroup group = resource.getResource(); for (Map.Entry entry : group.getQuotaMap().entrySet()) { @@ -399,7 +398,7 @@ public List> fetchProperty() { result.add(Lists.newArrayList(clusterPrefix + DppConfig.getPriorityKey(), String.valueOf(dppConfig.getPriority()))); } - + // get resolved ips if user has domain Map> resolvedIPs = whiteList.getResolvedIPs(); List ips = Lists.newArrayList(); @@ -410,9 +409,6 @@ public List> fetchProperty() { result.add(Lists.newArrayList("resolved IPs", Joiner.on(";").join(ips))); } - // bind_sql_block_rules - result.add(Lists.newArrayList(PROP_SQL_BLOCK_RULES, sqlBlockRules)); - // sort Collections.sort(result, new Comparator>() { @Override @@ -431,7 +427,6 @@ public static UserProperty read(DataInput in) throws IOException { } - @Override public void write(DataOutput out) throws IOException { // user name @@ -458,10 +453,6 @@ public void write(DataOutput out) throws IOException { // common properties commonProperties.write(out); - - if (StringUtils.isNotEmpty(sqlBlockRules)) { - Text.writeString(out, sqlBlockRules); - } } public void readFields(DataInput in) throws IOException { @@ -550,9 +541,5 @@ public void readFields(DataInput in) throws IOException { if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_100) { this.commonProperties = CommonUserProperties.read(in); } - - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_103) { - sqlBlockRules = Text.readString(in); - } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index 6594ede7cb33a4..c681d3808e6ea8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -220,10 +220,10 @@ public void addUserPrivEntriesByResolvedIPs(Map> resolvedIPs } } - public String getSqlBlockRules(String qualifiedUser) { + public String[] getSqlBlockRules(String qualifiedUser) { UserProperty existProperty = propertyMap.get(qualifiedUser); if (existProperty == null) { - return null; + return new String[]{}; } return existProperty.getSqlBlockRules(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java index 6fd1bf786bd8db..a98049da3af8c1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java @@ -113,5 +113,15 @@ public void testUpdate() throws DdlException { userProperty.update(properties); Assert.assertEquals(null, userProperty.getLoadClusterInfo("dpp-cluster").second); Assert.assertEquals(null, userProperty.getDefaultLoadCluster()); + + // sql block rule + properties.clear(); + properties.add(Pair.create("sql_block_rules", "")); + userProperty.update(properties); + Assert.assertEquals(1, userProperty.getSqlBlockRules().length); + properties.clear(); + properties.add(Pair.create("sql_block_rules", "test1, test2,test3")); + userProperty.update(properties); + Assert.assertEquals(3, userProperty.getSqlBlockRules().length); } } From 9caf01a43070b50f99efc4c884963bd3888ee91b Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 9 Aug 2021 10:22:25 +0800 Subject: [PATCH 09/14] MOD: sort code --- .../main/java/org/apache/doris/catalog/Catalog.java | 2 +- .../main/java/org/apache/doris/common/Config.java | 4 ++-- .../apache/doris/mysql/privilege/UserProperty.java | 9 ++++----- .../java/org/apache/doris/qe/ConnectProcessor.java | 12 +++++------- .../main/java/org/apache/doris/qe/StmtExecutor.java | 1 - 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 2ecdd60d2ea0c6..de30939a0ede33 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1935,7 +1935,7 @@ public void saveImage(File curFile, long replayedJournalId) throws IOException { MetaWriter.write(curFile, this); } - public long saveHeader(DataOutputStream dos, long replayedJournalId, long checksum) throws IOException { + public long saveHeader(CountingDataOutputStream dos, long replayedJournalId, long checksum) throws IOException { // Write meta version checksum ^= FeConstants.meta_version; dos.writeInt(FeConstants.meta_version); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index ed2751bd3feaff..95e89f85786f88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1469,12 +1469,12 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static int default_max_query_instances = -1; - /** + /* * One master daemon thread will update global partition in memory info every partition_in_memory_update_interval_secs */ @ConfField(mutable = false, masterOnly = true) public static int partition_in_memory_update_interval_secs = 300; - + @ConfField(masterOnly = true) public static boolean enable_concurrent_update = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index 3ee118c83e6f50..d756cc8f620e5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -250,6 +250,7 @@ public void update(List> properties) throws DdlException { throw new DdlException(PROP_SQL_BLOCK_RULES + " format error"); } sqlBlockRules = value; + this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); } else { throw new DdlException("Unknown user property(" + key + ")"); } @@ -259,7 +260,6 @@ public void update(List> properties) throws DdlException { this.commonProperties.setMaxConn(newMaxConn); this.commonProperties.setMaxQueryInstances(newMaxQueryInstances); this.commonProperties.setSqlBlockRules(sqlBlockRules); - this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); resource = newResource; if (newDppConfigs.containsKey(newDefaultLoadCluster)) { defaultLoadCluster = newDefaultLoadCluster; @@ -398,7 +398,7 @@ public List> fetchProperty() { result.add(Lists.newArrayList(clusterPrefix + DppConfig.getPriorityKey(), String.valueOf(dppConfig.getPriority()))); } - + // get resolved ips if user has domain Map> resolvedIPs = whiteList.getResolvedIPs(); List ips = Lists.newArrayList(); @@ -425,8 +425,7 @@ public static UserProperty read(DataInput in) throws IOException { userProperty.readFields(in); return userProperty; } - - + @Override public void write(DataOutput out) throws IOException { // user name @@ -460,7 +459,7 @@ public void readFields(DataInput in) throws IOException { // consume the flag of empty user name in.readBoolean(); } - + // user name if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_30) { qualifiedUser = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, Text.readString(in)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 62e82c7eb35b71..72c29931a6c085 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -50,8 +50,8 @@ import org.apache.doris.thrift.TMasterOpResult; import org.apache.doris.thrift.TUniqueId; -import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.google.common.base.Strings; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; @@ -144,10 +144,10 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue } ctx.getAuditEventBuilder().setFeIp(FrontendOptions.getLocalHostAddress()); - - String sql; + + // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { - sql = parsedStmt.toSql(); + ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql()); } else { if (parsedStmt instanceof InsertStmt && ((InsertStmt)parsedStmt).isValuesOrConstantSelect()) { // INSERT INTO VALUES may be very long, so we only log at most 1K bytes. @@ -156,10 +156,8 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue } else { ctx.getAuditEventBuilder().setStmt(origStmt); } - sql = origStmt; } - // We put origin query stmt at the end of audit log, for parsing the log more convenient. - ctx.getAuditEventBuilder().setStmt(sql); + Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 04e481cb60f5ca..49bd19ded540ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -1460,6 +1460,5 @@ public Data.PQueryStatistics getQueryStatisticsForAuditLog() { private List exprToType(List exprs) { return exprs.stream().map(e -> e.getType().getPrimitiveType()).collect(Collectors.toList()); } - } From 72363aee5e03cc9ac2550fa5b6b4f1c53ec9d2d3 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 9 Aug 2021 11:35:01 +0800 Subject: [PATCH 10/14] MOD: move sqlBlockRulesSplit to CommonUserProperties --- .../java/org/apache/doris/catalog/Catalog.java | 2 +- .../mysql/privilege/CommonUserProperties.java | 16 +++++++++++++++- .../doris/mysql/privilege/UserProperty.java | 13 +------------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index de30939a0ede33..0b49a6d8476601 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1905,7 +1905,7 @@ public long loadSmallFiles(DataInputStream in, long checksum) throws IOException } public long loadSqlBlockRule(DataInputStream in, long checksum) throws IOException { - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_103) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_104) { sqlBlockRuleMgr = SqlBlockRuleMgr.read(in); } LOG.info("finished replay sqlBlockRule from image"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index 605aee63a3f704..feec7f55b990f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -37,6 +37,7 @@ public class CommonUserProperties implements Writable { private long maxQueryInstances = -1; @SerializedName("sqlBlockRules") private String sqlBlockRules = ""; + private String[] sqlBlockRulesSplit; long getMaxConn() { return maxConn; @@ -49,6 +50,10 @@ long getMaxQueryInstances() { String getSqlBlockRules() { return sqlBlockRules; } + + String[] getSqlBlockRulesSplit() { + return sqlBlockRulesSplit; + } void setMaxConn(long maxConn) { this.maxConn = maxConn; @@ -60,11 +65,20 @@ void setMaxQueryInstances(long maxQueryInstances) { void setSqlBlockRules(String sqlBlockRules) { this.sqlBlockRules = sqlBlockRules; + setSqlBlockRulesSplit(sqlBlockRules); + } + + void setSqlBlockRulesSplit(String sqlBlockRules) { + // split + this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); } public static CommonUserProperties read(DataInput in) throws IOException { String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, CommonUserProperties.class); + CommonUserProperties commonUserProperties = GsonUtils.GSON.fromJson(json, CommonUserProperties.class); + // trigger split + commonUserProperties.setSqlBlockRulesSplit(commonUserProperties.getSqlBlockRules()); + return commonUserProperties; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index d756cc8f620e5c..f3a7867ad8eb4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -90,9 +90,6 @@ public class UserProperty implements Writable { */ private WhiteList whiteList = new WhiteList(); - // the binding of sql_block_rule name, multiple are separated by ',' - private String[] sqlBlockRulesSplit = {}; - static { ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_MAX_USER_CONNECTIONS + "$", Pattern.CASE_INSENSITIVE)); ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_RESOURCE + ".", Pattern.CASE_INSENSITIVE)); @@ -127,14 +124,7 @@ public long getMaxQueryInstances() { } public String[] getSqlBlockRules() { - if (this.sqlBlockRulesSplit.length != 0) { - return this.sqlBlockRulesSplit; - } - String sqlBlockRules = commonProperties.getSqlBlockRules(); - if (StringUtils.isNotEmpty(sqlBlockRules)) { - this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); - } - return this.sqlBlockRulesSplit; + return commonProperties.getSqlBlockRulesSplit(); } public WhiteList getWhiteList() { @@ -250,7 +240,6 @@ public void update(List> properties) throws DdlException { throw new DdlException(PROP_SQL_BLOCK_RULES + " format error"); } sqlBlockRules = value; - this.sqlBlockRulesSplit = sqlBlockRules.replace(" ", "").split(","); } else { throw new DdlException("Unknown user property(" + key + ")"); } From 4ffc55fb70a7917346353fd817e49b12c5711ec0 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 9 Aug 2021 14:51:47 +0800 Subject: [PATCH 11/14] FIX: npe --- .../org/apache/doris/mysql/privilege/CommonUserProperties.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index feec7f55b990f9..8ab5ebee68e06b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -37,7 +37,7 @@ public class CommonUserProperties implements Writable { private long maxQueryInstances = -1; @SerializedName("sqlBlockRules") private String sqlBlockRules = ""; - private String[] sqlBlockRulesSplit; + private String[] sqlBlockRulesSplit = {}; long getMaxConn() { return maxConn; From fa6e47edf867a013be2521fd7329faca9da68b9f Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 9 Aug 2021 16:21:03 +0800 Subject: [PATCH 12/14] ADD: ut --- .../doris/blockrule/SqlBlockRuleMgrTest.java | 110 +++++++++++++----- 1 file changed, 79 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java index 0a7b054e0e98f8..8fd7af294a6138 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java @@ -17,47 +17,99 @@ package org.apache.doris.blockrule; -import org.apache.doris.analysis.AccessTestUtil; -import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateSqlBlockRuleStmt; +import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.SetUserPropertyStmt; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.metric.MetricRepo; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.UtFrameUtils; import org.apache.commons.codec.digest.DigestUtils; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; - -import mockit.Mocked; +import java.io.File; +import java.util.UUID; public class SqlBlockRuleMgrTest { - - private Analyzer analyzer; - - @Before - public void setUp() { - analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + + private static String runningDir = "fe/mocked/SqlBlockRuleMgrTest/" + UUID.randomUUID().toString() + "/"; + + private static ConnectContext connectContext; + + @BeforeClass + public static void beforeClass() throws Exception { + UtFrameUtils.createMinDorisCluster(runningDir); + + // create connect context + connectContext = UtFrameUtils.createDefaultCtx(); + + // create database + String createDbStmtStr = "create database test;"; + CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext); + Catalog.getCurrentCatalog().createDb(createDbStmt); + MetricRepo.init(); - + createTable("create table test.table1\n" + + "(k1 int, k2 int) distributed by hash(k1) buckets 1\n" + + "properties(\"replication_num\" = \"1\");"); + } - - - @Test(expected = AnalysisException.class) + + @AfterClass + public static void tearDown() { + File file = new File(runningDir); + file.delete(); + } + + private static void createTable(String sql) throws Exception { + CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); + Catalog.getCurrentCatalog().createTable(createTableStmt); + } + + @Test + public void testUserMatchSql() throws Exception { + String sql = "select * from table1 limit 10"; + String sqlHash = DigestUtils.md5Hex(sql); + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, sqlHash, false, true); + SqlBlockRuleMgr mgr = new SqlBlockRuleMgr(); + mgr.replayCreate(sqlRule); + // sql block rules + String setPropertyStr = "set property for \"root\" \"sql_block_rules\" = \"test_rule1\""; + SetUserPropertyStmt setUserPropertyStmt = (SetUserPropertyStmt) UtFrameUtils.parseAndAnalyzeStmt(setPropertyStr, connectContext); + Catalog.getCurrentCatalog().getAuth().updateUserProperty(setUserPropertyStmt); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "sql match hash sql block rule: " + sqlRule.getName(), + () -> mgr.matchSql(sql, sqlHash, "root")); + } + + @Test + public void testGlobalMatchSql() throws AnalysisException { + String sql = "select * from test_table1 limit 10"; + String sqlHash = DigestUtils.md5Hex(sql); + SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, sqlHash, true, true); + SqlBlockRuleMgr mgr = new SqlBlockRuleMgr(); + mgr.replayCreate(sqlRule); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "sql match hash sql block rule: " + sqlRule.getName(), + () -> mgr.matchSql(sql, sqlHash, "test")); + } + + @Test public void testRegexMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; String sqlHash = DigestUtils.md5Hex(sql); SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", ".* join .*", null, true, true); SqlBlockRuleMgr mgr = new SqlBlockRuleMgr(); mgr.replayCreate(sqlRule); - mgr.matchSql(sqlRule, sql, sqlHash); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "sql match regex sql block rule: " + sqlRule.getName(), + () -> mgr.matchSql(sqlRule, sql, sqlHash)); } - - @Test(expected = AnalysisException.class) + + @Test public void testHashMatchSql() throws AnalysisException { String sql = "select * from test_table1 tt1 join test_table2 tt2 on tt1.testId=tt2.testId limit 5"; String sqlHash = DigestUtils.md5Hex(sql); @@ -65,17 +117,13 @@ public void testHashMatchSql() throws AnalysisException { SqlBlockRule sqlRule = new SqlBlockRule("test_rule1", null, sqlHash, true, true); SqlBlockRuleMgr mgr = new SqlBlockRuleMgr(); mgr.replayCreate(sqlRule); - mgr.matchSql(sqlRule, sql, sqlHash); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "sql match hash sql block rule: " + sqlRule.getName(), + () -> mgr.matchSql(sqlRule, sql, sqlHash)); } - + @Test - public void testNormalCreate(@Mocked ConnectContext connectContext, @Mocked Catalog catalog) throws UserException { - SqlBlockRuleMgr sqlBlockRuleMgr = new SqlBlockRuleMgr(); - Map properties = new HashMap<>(); - properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); - properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); - CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); - stmt.analyze(analyzer); - sqlBlockRuleMgr.createSqlBlockRule(stmt); + public void testNormalCreate() throws Exception { + String createSql = "CREATE SQL_BLOCK_RULE test_rule PROPERTIES(\"sql\"=\"select \\\\* from test_table\",\"enable\"=\"true\")"; + CreateSqlBlockRuleStmt createSqlBlockRuleStmt = (CreateSqlBlockRuleStmt) UtFrameUtils.parseAndAnalyzeStmt(createSql, connectContext); } } From e2bd90777efd0c691280a11a19bc53a88d323ab4 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 9 Aug 2021 21:17:18 +0800 Subject: [PATCH 13/14] FIX: fix ut --- .../doris/analysis/CreateSqlBlockRuleStmtTest.java | 1 - .../test/java/org/apache/doris/qe/ShowExecutorTest.java | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java index e4a6beaa8d52d5..f2cb284d4cb58e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateSqlBlockRuleStmtTest.java @@ -66,7 +66,6 @@ public void testNormal() throws UserException { @Test(expected = AnalysisException.class) public void testNoProps() throws UserException { Map properties = new HashMap<>(); - properties.put(CreateSqlBlockRuleStmt.SQL_PROPERTY, "select \\* from test_table"); properties.put(CreateSqlBlockRuleStmt.ENABLE_PROPERTY, "true"); CreateSqlBlockRuleStmt stmt = new CreateSqlBlockRuleStmt("test_rule", properties); stmt.analyze(analyzer); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java index 51cc6705e46a97..bbb71108e5c2f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java @@ -596,10 +596,11 @@ public void testShowSqlBlockRule() throws AnalysisException { ShowSqlBlockRuleStmt stmt = new ShowSqlBlockRuleStmt("test_rule"); ShowExecutor executor = new ShowExecutor(ctx, stmt); ShowResultSet resultSet = executor.execute(); - Assert.assertEquals(4, resultSet.getMetaData().getColumnCount()); + Assert.assertEquals(5, resultSet.getMetaData().getColumnCount()); Assert.assertEquals("Name", resultSet.getMetaData().getColumn(0).getName()); - Assert.assertEquals("User", resultSet.getMetaData().getColumn(1).getName()); - Assert.assertEquals("Sql", resultSet.getMetaData().getColumn(2).getName()); - Assert.assertEquals("Enable", resultSet.getMetaData().getColumn(3).getName()); + Assert.assertEquals("Sql", resultSet.getMetaData().getColumn(1).getName()); + Assert.assertEquals("SqlHash", resultSet.getMetaData().getColumn(2).getName()); + Assert.assertEquals("Global", resultSet.getMetaData().getColumn(3).getName()); + Assert.assertEquals("Enable", resultSet.getMetaData().getColumn(4).getName()); } } From a4b6d4968fdd72ebe8b6f15aef218393350388a2 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 9 Aug 2021 21:53:44 +0800 Subject: [PATCH 14/14] MOD: clean code --- .../src/main/java/org/apache/doris/analysis/UserIdentity.java | 4 ---- .../src/main/java/org/apache/doris/qe/ConnectProcessor.java | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java index 03a457eb086b22..51822d1c75cfe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java @@ -97,10 +97,6 @@ public String getQualifiedUser() { return user; } - public String getUser() { - return user; - } - public String getHost() { return host; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 72c29931a6c085..056b8902a49ec4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -144,7 +144,7 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue } ctx.getAuditEventBuilder().setFeIp(FrontendOptions.getLocalHostAddress()); - + // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql()); @@ -157,7 +157,7 @@ private void auditAfterExec(String origStmt, StatementBase parsedStmt, Data.PQue ctx.getAuditEventBuilder().setStmt(origStmt); } } - + Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); }