1.1.1 rc03 (#11161)

* [fix](fe_plugins) update fe plugins' dependency to fe 1.0-SNAPSHOT (#11141) * 1.1.1 rc03 Co-authored-by: Mingyu Chen <morningman.cmy@gmail.com> Co-authored-by: yiguolei <yiguolei@gmail.com>
apache · Jul 25, 2022 · 2dbd70b · 2dbd70b
1 parent a894905
commit 2dbd70b
Show file tree

Hide file tree

Showing 7 changed files with 223 additions and 5 deletions.
diff --git a/docs/en/docs/ecosystem/audit-plugin.md b/docs/en/docs/ecosystem/audit-plugin.md
@@ -0,0 +1,101 @@
+---
+{
+    "title": "Audit log plugin",
+    "language": "en"
+}
+---
+
+<!-
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+->
+
+# Audit log plugin
+
+Doris's audit log plugin was developed based on FE's plugin framework. Is an optional plugin. Users can install or uninstall this plugin at runtime.
+
+This plugin can periodically import the FE audit log into the specified Doris cluster, so that users can easily view and analyze the audit log through SQL.
+
+## Compile, Configure and Deploy
+
+### FE Configuration
+
+FE's plugin framework is an experimental feature, which is closed by default. In the FE configuration file, add `plugin_enable = true` to enable the plugin framework.
+
+### AuditLoader Configuration
+
+The configuration of the auditloader plugin is located in `$ {DORIS}/fe_plugins/auditloader/src/main/assembly/`.
+
+Open `plugin.conf` for configuration. See the comments of the configuration items.
+
+### Compile
+
+After executing `sh build_plugin.sh` in the Doris code directory, you will get the `auditloader.zip` file in the `fe_plugins/output` directory.
+
+### Deployment
+
+You can place this file on an http download server or copy(or unzip) it to the specified directory of all FEs. Here we use the latter.
+
+### Installation
+
+After deployment is complete, and before installing the plugin, you need to create the audit database and tables previously specified in `plugin.conf`. The table creation statement is as follows:
+
+```
+create table doris_audit_tbl__
+(
+    query_id varchar(48) comment "Unique query id",
+    `time` datetime not null comment "Query start time",
+    client_ip varchar(32) comment "Client IP",
+    user varchar(64) comment "User name",
+    db varchar(96) comment "Database of this query",
+    state varchar(8) comment "Query result state. EOF, ERR, OK",
+    query_time bigint comment "Query execution time in millisecond",
+    scan_bytes bigint comment "Total scan bytes of this query",
+    scan_rows bigint comment "Total scan rows of this query",
+    return_rows bigint comment "Returned rows of this query",
+    stmt_id int comment "An incremental id of statement",
+    is_query tinyint comment "Is this statemt a query. 1 or 0",
+    frontend_ip varchar(32) comment "Frontend ip of executing this statement",
+    cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
+    sql_hash varchar(50) comment "Hash value for this query",
+    sql_digest varchar(48) comment "Sql digest for this query",
+    peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
+    stmt string comment "The original statement, trimed if longer than 2G"
+) engine=OLAP
+duplicate key(query_id, `time`, client_ip)
+partition by range(`time`) ()
+distributed by hash(query_id) buckets 1
+properties(
+    "dynamic_partition.time_unit" = "DAY",
+    "dynamic_partition.start" = "-30",
+    "dynamic_partition.end" = "3",
+    "dynamic_partition.prefix" = "p",
+    "dynamic_partition.buckets" = "1",
+    "dynamic_partition.enable" = "true",
+    "replication_num" = "3"
+);
+```
+
+>**Notice**
+>
+> In the above table structure: stmt string, this can only be used in 0.15 and later versions, in previous versions, the field type used varchar
+
+The `dynamic_partition` attribute selects the number of days to keep the audit log based on your needs.
+
+After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`.
+
+Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
diff --git a/docs/zh-CN/docs/ecosystem/audit-plugin.md b/docs/zh-CN/docs/ecosystem/audit-plugin.md
@@ -0,0 +1,101 @@
+---
+{
+    "title": "审计日志插件",
+    "language": "zh-CN"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# 审计日志插件
+
+Doris 的审计日志插件是在 FE 的插件框架基础上开发的。是一个可选插件。用户可以在运行时安装或卸载这个插件。
+
+该插件可以将 FE 的审计日志定期的导入到指定 Doris 集群中，以方便用户通过 SQL 对审计日志进行查看和分析。
+
+## 编译、配置和部署
+
+### FE 配置
+
+FE的插件框架当前是实验性功能，Doris中默认关闭，在FE的配置文件中，增加`plugin_enable = true`启用plugin框架
+
+### AuditLoader 配置
+
+auditloader plugin的配置位于`${DORIS}/fe_plugins/auditloader/src/main/assembly/`.
+
+打开 `plugin.conf` 进行配置。配置项说明参见注释。
+
+### 编译
+
+在 Doris 代码目录下执行 `sh build_plugin.sh` 后，会在 `fe_plugins/output` 目录下得到 `auditloader.zip` 文件。
+
+### 部署
+
+您可以将这个文件放置在一个 http 服务器上，或者拷贝`auditloader.zip`(或者解压`auditloader.zip`)到所有 FE 的指定目录下。这里我们使用后者。
+
+### 安装
+
+部署完成后，安装插件前，需要创建之前在 `plugin.conf` 中指定的审计数据库和表。其中建表语句如下：
+
+```
+create table doris_audit_tbl__
+(
+    query_id varchar(48) comment "Unique query id",
+    `time` datetime not null comment "Query start time",
+    client_ip varchar(32) comment "Client IP",
+    user varchar(64) comment "User name",
+    db varchar(96) comment "Database of this query",
+    state varchar(8) comment "Query result state. EOF, ERR, OK",
+    query_time bigint comment "Query execution time in millisecond",
+    scan_bytes bigint comment "Total scan bytes of this query",
+    scan_rows bigint comment "Total scan rows of this query",
+    return_rows bigint comment "Returned rows of this query",
+    stmt_id int comment "An incremental id of statement",
+    is_query tinyint comment "Is this statemt a query. 1 or 0",
+    frontend_ip varchar(32) comment "Frontend ip of executing this statement",
+    cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
+    sql_hash varchar(48) comment "Hash value for this query",
+    sql_digest varchar(48) comment "Sql digest for this query",
+    peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
+    stmt string comment "The original statement, trimed if longer than 2G "
+) engine=OLAP
+duplicate key(query_id, `time`, client_ip)
+partition by range(`time`) ()
+distributed by hash(query_id) buckets 1
+properties(
+    "dynamic_partition.time_unit" = "DAY",
+    "dynamic_partition.start" = "-30",
+    "dynamic_partition.end" = "3",
+    "dynamic_partition.prefix" = "p",
+    "dynamic_partition.buckets" = "1",
+    "dynamic_partition.enable" = "true",
+    "replication_num" = "3"
+);
+```
+
+>**注意**
+>
+> 上面表结构中：stmt string ，这个只能在0.15及之后版本中使用，之前版本，字段类型使用varchar
+
+其中 `dynamic_partition` 属性根据自己的需要，选择审计日志保留的天数。
+
+之后，连接到 Doris 后使用 `INSTALL PLUGIN` 命令完成安装。安装成功后，可以通过 `SHOW PLUGINS` 看到已经安装的插件，并且状态为 `INSTALLED`。
+
+完成后，插件会不断的以指定的时间间隔将审计日志插入到这个表中。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DigitalVersion.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DigitalVersion.java
@@ -81,7 +81,8 @@ public static DigitalVersion fromString(String version) throws IllegalArgumentEx
             int revision = Integer.parseInt(list[2].trim());
 
             if (major >= 100 || minor >= 100 || revision >= 100) {
-                throw new IllegalArgumentException("Illegal version format: " + version);
+                throw new IllegalArgumentException(
+                        "Illegal version format: " + version + ". Expected: major.minor.revision");
             }
 
             return new DigitalVersion((byte) major, (byte) minor, (byte) revision);

diff --git a/fe_plugins/auditloader/src/main/assembly/plugin.properties b/fe_plugins/auditloader/src/main/assembly/plugin.properties
@@ -18,6 +18,6 @@
 name=AuditLoader
 type=AUDIT
 description=load audit log to olap load, and user can view the statistic of queries
-version=0.13.1
+version=1.0.0
 java.version=1.8.0
 classname=org.apache.doris.plugin.audit.AuditLoaderPlugin
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
@@ -69,7 +69,7 @@ private HttpURLConnection getConnection(String urlStr, String label) throws IOEx
 
         conn.addRequestProperty("label", label);
         conn.addRequestProperty("max_filter_ratio", "1.0");
-        conn.addRequestProperty("columns", "query_id, time, client_ip, user, db, state, query_time, scan_bytes," +
+        conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, query_time, scan_bytes," +
                 " scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, peak_memory_bytes, stmt");
 
         conn.setDoOutput(true);

diff --git a/fe_plugins/pom.xml b/fe_plugins/pom.xml
@@ -68,7 +68,7 @@ under the License.
     </modules>
     <properties>
         <log4j2.version>2.17.1</log4j2.version>
-        <doris.version>0.15-SNAPSHOT</doris.version>
+        <doris.version>1.0-SNAPSHOT</doris.version>
         <project.scm.id>github</project.scm.id>
     </properties>
     <profiles>
@@ -93,6 +93,21 @@ under the License.
                 </pluginRepository>
             </pluginRepositories>
         </profile>
+        <profile>
+            <id>general-env</id>
+            <activation>
+                <property>
+                    <name>!env.CUSTOM_MAVEN_REPO</name>
+                </property>
+            </activation>
+            <repositories>
+                <!-- for java-cup -->
+                <repository>
+                    <id>cloudera-public</id>
+                    <url>https://repository.cloudera.com/artifactory/public/</url>
+                </repository>
+            </repositories>
+        </profile>
     </profiles>
     <dependencyManagement>
         <dependencies>

diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh
@@ -25,7 +25,7 @@
 # contains the build version based on the git hash or svn revision.
 ##############################################################
 
-build_version="1.1.1-rc02"
+build_version="1.1.1-rc03"
 
 unset LANG
 unset LC_CTYPE