feat(config): support new config & Feat config listener (#374)

* feat: add group api for UI module #344 (#373) * add: missing interfaces in proto files and discovery files * feat: group api in file db_groups.go (#344) * style: add license header and add tool: imports-formatter * style: fix import * feat: support notify event * refactor:config center * none * feat: none * refactor:split config and add change watch * style: fix code style * fix: fix func call error * fix: ci error * style:fix code style * fix: fix code style * refactor: rebase upstream * fix:add cluster info to list groups Co-authored-by: chovychan <51713304+chovychan@users.noreply.github.com> Co-authored-by: Jeffsky <jjeffcaii@outlook.com>
arana-db · Sep 4, 2022 · 6ae534f · 6ae534f
1 parent b0a9024
commit 6ae534f
Show file tree

Hide file tree

Showing 38 changed files with 3,055 additions and 1,327 deletions.
diff --git a/cmd/start/start.go b/cmd/start/start.go
@@ -33,7 +33,6 @@ import (
 import (
 	"github.com/arana-db/arana/cmd/cmds"
 	"github.com/arana-db/arana/pkg/boot"
-	"github.com/arana-db/arana/pkg/config"
 	"github.com/arana-db/arana/pkg/constants"
 	"github.com/arana-db/arana/pkg/executor"
 	"github.com/arana-db/arana/pkg/mysql"
@@ -85,13 +84,7 @@ func Run(bootstrapConfigPath string, importPath string) {
 	}
 
 	if len(importPath) > 0 {
-		c, err := config.Load(importPath)
-		if err != nil {
-			log.Fatal("failed to import configuration from %s: %v", importPath, err)
-			return
-		}
-		if err := discovery.GetConfigCenter().ImportConfiguration(c); err != nil {
-			log.Fatal("failed to import configuration from %s: %v", importPath, err)
+		if !boot.RunImport(bootstrapConfigPath, importPath) {
 			return
 		}
 	}
@@ -103,12 +96,7 @@ func Run(bootstrapConfigPath string, importPath string) {
 
 	propeller := server.NewServer()
 
-	listenersConf, err := discovery.ListListeners(context.Background())
-	if err != nil {
-		log.Fatal("start failed: %v", err)
-		return
-	}
-
+	listenersConf := discovery.ListListeners(context.Background())
 	for _, listenerConf := range listenersConf {
 		listener, err := mysql.NewListener(listenerConf)
 		if err != nil {

diff --git a/cmd/tools/tools.go b/cmd/tools/tools.go
@@ -18,7 +18,7 @@
 package tools
 
 import (
-	"context"
+	"github.com/arana-db/arana/pkg/boot"
 	"os"
 )
 
@@ -28,10 +28,7 @@ import (
 
 import (
 	"github.com/arana-db/arana/cmd/cmds"
-	"github.com/arana-db/arana/pkg/boot"
-	"github.com/arana-db/arana/pkg/config"
 	"github.com/arana-db/arana/pkg/constants"
-	"github.com/arana-db/arana/pkg/util/log"
 )
 
 var (
@@ -45,7 +42,7 @@ func init() {
 		Use:     "import",
 		Short:   "import arana config",
 		Example: "./arana import -c ../docker/conf/bootstrap.yaml -s ../docker/conf/config.yaml",
-		Run:     Run,
+		Run:     run,
 	}
 
 	cmd.PersistentFlags().
@@ -58,25 +55,10 @@ func init() {
 	})
 }
 
-func Run(cmd *cobra.Command, args []string) {
-	_, _ = cmd, args
-
-	discovery := boot.NewDiscovery(importBootConfPath)
-	if err := discovery.Init(context.Background()); err != nil {
-		log.Fatal("init failed: %+v", err)
-		return
-	}
-
-	cfg, err := config.Load(sourceConfigPath)
-	if err != nil {
-		log.Fatal("load config from %s failed: %+v", sourceConfigPath, err)
-		return
-	}
-
-	c := discovery.GetConfigCenter()
+func run(_ *cobra.Command, _ []string) {
+	Run(importBootConfPath, sourceConfigPath)
+}
 
-	if err := c.ImportConfiguration(cfg); err != nil {
-		log.Fatal("persist config to config.store failed: %+v", err)
-		return
-	}
+func Run(importConfPath, configPath string) {
+	boot.RunImport(importConfPath, configPath)
 }
diff --git a/conf/bootstrap.yaml b/conf/bootstrap.yaml
@@ -15,19 +15,29 @@
 # limitations under the License.
 #
 
+kind: ConfigMap
+apiVersion: "1.0"
+listeners:
+  - protocol_type: mysql
+    server_version: 5.7.0
+    socket_address:
+      address: 0.0.0.0
+      port: 13306
 config:
   name: file
   options:
 
-  # name: etcd
-  # options:
-  #   endpoints: "http://localhost:2379"
-  # name: nacos
-  # options:
-  #   endpoints: "localhost:8080"
-  #   namespace: arana
-  #   group: arana
-  #   contextPath: /nacos
-  #   scheme: http
-  #   username: nacos
-  #   password: nacos
+#  name: etcd
+#  root_path: arana
+#  options:
+#    endpoints: "http://127.0.0.1:2379"
+
+#   name: nacos
+#   options:
+#     endpoints: "127.0.0.1:8848"
+#     namespace: arana
+#     group: arana
+#     contextPath: /nacos
+#     scheme: http
+#     username: nacos
+#     password: nacos
diff --git a/conf/config.yaml b/conf/config.yaml
@@ -20,121 +20,99 @@ apiVersion: "1.0"
 metadata:
   name: arana-config
 data:
-  listeners:
-    - protocol_type: mysql
-      server_version: 5.7.0
-      socket_address:
-        address: 0.0.0.0
-        port: 13306
-
   tenants:
     - name: arana
       users:
         - username: root
           password: "123456"
         - username: arana
           password: "123456"
-
-  clusters:
-    - name: employees
-      type: mysql
-      sql_max_limit: -1
-      tenant: arana
-      parameters:
-        max_allowed_packet: 256M
-      groups:
-        - name: employees_0000
-          nodes:
-            - name: node0
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_0000
-              weight: r10w10
-              parameters:
-            - name: node0_r_0
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_0000_r
-              weight: r0w0
-        - name: employees_0001
-          nodes:
-            - name: node1
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_0001
-              weight: r10w10
-        - name: employees_0002
-          nodes:
-            - name: node2
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_0002
-              weight: r10w10
-        - name: employees_0003
-          nodes:
-            - name: node3
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_0003
-              weight: r10w10
-        - name: employees_shadow
-          nodes:
-            - name: node_shadow
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_show
-              weight: r10w10
-  sharding_rule:
-    tables:
-      - name: employees.student
-        allow_full_scan: true
-        sequence:
-          type: snowflake
-          option:
-        db_rules:
-          - column: uid
-            type: scriptExpr
-            expr: parseInt($value % 32 / 8)
-        tbl_rules:
-          - column: uid
-            type: scriptExpr
-            expr: $value % 32
-            step: 32
-        topology:
-          db_pattern: employees_${0000..0003}
-          tbl_pattern: student_${0000..0031}
-        attributes:
-          sqlMaxLimit: -1
-
-  shadow_rule:
-    tables:
-      - name: student
-        enable: false
-        group_node: employees_shadow
-        match_rules:
-          - operation: [insert,update]
-            match_type: value
-            attributes:
+      clusters:
+        - name: employees
+          type: mysql
+          sql_max_limit: -1
+          tenant: arana
+          parameters:
+            max_allowed_packet: 256M
+          groups:
+            - name: employees_0000
+              nodes:
+                - node0
+                - node0_r_0
+            - name: employees_0001
+              nodes:
+                - node1
+            - name: employees_0002
+              nodes:
+                - node2
+            - name: employees_0003
+              nodes:
+                - node3
+      sharding_rule:
+        tables:
+          - name: employees.student
+            allow_full_scan: true
+            sequence:
+              type: snowflake
+              option:
+            db_rules:
               - column: uid
-                value: 10000
-          - operation: [delete]
-            match_type: regex
-            attributes:
-              - column: name
-                regex: "^hanmeimei$"
-          - operation: [select]
-            match_type: hint
+                type: scriptExpr
+                expr: parseInt($value % 32 / 8)
+            tbl_rules:
+              - column: uid
+                type: scriptExpr
+                expr: $value % 32
+                step: 32
+            topology:
+              db_pattern: employees_${0000..0003}
+              tbl_pattern: student_${0000..0031}
             attributes:
-              - shadow: true
+              sqlMaxLimit: -1
+      nodes:
+        node0:
+          name: node0
+          host: arana-mysql
+          port: 3306
+          username: root
+          password: "123456"
+          database: employees_0000
+          weight: r10w10
+          parameters:
+        node0_r_0:
+          name: node0_r_0
+          host: arana-mysql
+          port: 3306
+          username: root
+          password: "123456"
+          database: employees_0000_r
+          weight: r0w0
+          parameters:
+        node1:
+          name: node1
+          host: arana-mysql
+          port: 3306
+          username: root
+          password: "123456"
+          database: employees_0001
+          weight: r10w10
+          parameters:
+        node2:
+          name: node2
+          host: arana-mysql
+          port: 3306
+          username: root
+          password: "123456"
+          database: employees_0002
+          weight: r10w10
+          parameters:
+        node3:
+          name: node3
+          host: arana-mysql
+          port: 3306
+          username: root
+          password: "123456"
+          database: employees_0003
+          weight: r10w10
+          parameters:
+
diff --git a/example/import_config/main.go b/example/import_config/main.go
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"github.com/arana-db/arana/cmd/tools"
+	"github.com/arana-db/arana/testdata"
+)
+
+func main() {
+	bootstrap := testdata.Path("../conf/bootstrap.yaml")
+	config := testdata.Path("../conf/config.yaml")
+	tools.Run(bootstrap, config)
+}