1. Defalut Null 异常修复

2. Create_Time CreateDate 兼容分区 3. 去除mongo Clickhouse docker多余镜像 4. 可以为NULL字段自动添加Nullable 设置
zeromicro · May 18, 2022 · 5e6ead3 · 5e6ead3
1 parent 4a237dc
commit 5e6ead3
Show file tree

Hide file tree

Showing 9 changed files with 157 additions and 147 deletions.
diff --git a/cmd/dm/util/mysqltypeconv.go b/cmd/dm/util/mysqltypeconv.go
@@ -16,12 +16,12 @@ type (
 		mysqlConn *sql.DB
 	}
 	MysqlDesc struct {
-		Field   string `db:"Field"`
-		Type    string `db:"Type"`
-		Null    string `db:"Null"`
-		Key     string `db:"Key"`
-		Default string `db:"Default"`
-		Extra   string `db:"Extra"`
+		Field   string         `db:"Field"`
+		Type    string         `db:"Type"`
+		Null    string         `db:"Null"`
+		Key     string         `db:"Key"`
+		Default sql.NullString `db:"Default"`
+		Extra   string         `db:"Extra"`
 	}
 	DataType int
 )
@@ -52,8 +52,11 @@ func (tc *MysqlTypeConverter) ObtainMysqlTypeMap(tableName string) (map[string]D
 	}
 	pks := make(map[string]int, 8)
 	index := 0
+	logx.Info("ObtainMysqlTypeMap tableName", tableName)
+	logx.Info("ObtainMysqlTypeMap rows", rows)
 	for rows.Next() {
 		var mysqlDesc MysqlDesc
+		logx.Info("ObtainMysqlTypeMap mysqlDesc.Default", mysqlDesc.Default, " &mysqlDesc.Default:", &mysqlDesc.Default)
 		err := rows.Scan(&mysqlDesc.Field, &mysqlDesc.Type, &mysqlDesc.Null, &mysqlDesc.Key, &mysqlDesc.Default, &mysqlDesc.Extra)
 		if err != nil {
 			logx.Error(err)

diff --git a/cmd/galaxy/etc/galaxy-api.yaml b/cmd/galaxy/etc/galaxy-api.yaml
@@ -13,7 +13,7 @@ HeraCk: tcp://127.0.0.1:9000?username=default&password=
 CkDataNodes:
   - tcp://ch-server-1:9000?username=default&password=
   - tcp://ch-server-1:9000?username=default&password=,tcp://ch-server-2:9000?username=default&password=
-  - tcp://ch-server-3:9000?username=default&password=,tcp://ch-server-4:9000?username=default&password=
+#  - tcp://ch-server-3:9000?username=default&password=,tcp://ch-server-4:9000?username=default&password=
 Debezium: kafka-connector:8083
 CanalConfig:
   UserName: admin

diff --git a/pkg/ckgroup/docker/config.xml b/pkg/ckgroup/docker/config.xml
@@ -12,17 +12,17 @@
                     <port>9000</port>
                 </replica>
             </shard>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>ch-server-3</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>ch-server-4</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
+<!--            <shard>-->
+<!--                <internal_replication>true</internal_replication>-->
+<!--                <replica>-->
+<!--                    <host>ch-server-3</host>-->
+<!--                    <port>9000</port>-->
+<!--                </replica>-->
+<!--                <replica>-->
+<!--                    <host>ch-server-4</host>-->
+<!--                    <port>9000</port>-->
+<!--                </replica>-->
+<!--            </shard>-->
         </bip_ck_cluster>
     </remote_servers>
     <zookeeper>

diff --git a/pkg/ckgroup/docker/docker-compose.yaml b/pkg/ckgroup/docker/docker-compose.yaml
@@ -22,28 +22,28 @@ services:
       - 9001:9000
     environment:
       TZ: "Asia/Shanghai"
-  ch-server-3:
-    image: yandex/clickhouse-server
-    volumes:
-      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
-      - ./users.xml:/etc/clickhouse-server/users.xml
-      - ./macros3.xml:/etc/clickhouse-server/config.d/macros.xml
-    ports:
-      - 8125:8123
-      - 9002:9000
-    environment:
-      TZ: "Asia/Shanghai"
-  ch-server-4:
-    image: yandex/clickhouse-server
-    volumes:
-      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
-      - ./users.xml:/etc/clickhouse-server/users.xml
-      - ./macros4.xml:/etc/clickhouse-server/config.d/macros.xml
-    ports:
-      - 8126:8123
-      - 9003:9000
-    environment:
-      TZ: "Asia/Shanghai"
+#  ch-server-3:
+#    image: yandex/clickhouse-server
+#    volumes:
+#      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
+#      - ./users.xml:/etc/clickhouse-server/users.xml
+#      - ./macros3.xml:/etc/clickhouse-server/config.d/macros.xml
+#    ports:
+#      - 8125:8123
+#      - 9002:9000
+#    environment:
+#      TZ: "Asia/Shanghai"
+#  ch-server-4:
+#    image: yandex/clickhouse-server
+#    volumes:
+#      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
+#      - ./users.xml:/etc/clickhouse-server/users.xml
+#      - ./macros4.xml:/etc/clickhouse-server/config.d/macros.xml
+#    ports:
+#      - 8126:8123
+#      - 9003:9000
+#    environment:
+#      TZ: "Asia/Shanghai"
   zookeeper:
     image: zookeeper
     environment:

diff --git a/pkg/mysqlx/clickhouse.go b/pkg/mysqlx/clickhouse.go
@@ -21,7 +21,7 @@ func ToClickhouseTable(dsn, db, table, indexes string, withTime bool) ([]string,
 		if c.Key == "PRI" {
 			pri = c.Field
 		}
-		if c.Field == "create_time" || c.Field == "createTime" {
+		if c.Field == "create_time" || c.Field == "createTime" || c.Field == "createdDate" {
 			createTime = c.Field
 		}
 		if c.Field == "update_time" || c.Field == "updateTime" || c.Field == "insert_id" {
@@ -30,7 +30,7 @@ func ToClickhouseTable(dsn, db, table, indexes string, withTime bool) ([]string,
 		// type converter
 		columns[i].Type = toClickhouseType(c.Type)
 		// 如果字段类型可以为null 则添加 Nullable,否则同步mysql回报nil异常
-		if c.Null == "YES" {
+		if c.Null == "YES" && c.Field != "createdDate" {
 			columns[i].Type = "Nullable(" + columns[i].Type + ")"
 		}
 		newColumns = append(newColumns, table2.Column{

diff --git a/script/init/init_db.py b/script/init/init_db.py
@@ -6,6 +6,9 @@
 import string
 import datetime
 
+from pymongo import MongoClient
+from pymongo.database import Database
+
 
 def init_mysql():
     conn = pymysql.connect(
@@ -50,26 +53,29 @@ def init_mysql():
     print("mysql `test_mysql`.`example` insert end")
 
 def init_mongo():
-    cli = pymongo.MongoClient(host="localhost")
-    db = cli["test_mongo"]
+    # cli = pymongo.MongoClient(host="mongo1", port=30001, connect=True)
+    # Database(cli, u'test_mongo')
+    uri = 'mongodb://mongo1:30001/'
+    client = MongoClient(uri)
+    # >>> db = client.get_default_database()
+    db = client["test_mongo"]
     collection = db["example"]
     result = []
 
     for i in range(1, 100001):
         result.append(
                 {
                     "float": random.random(), 
-                    "string":"".join(random.choices(string.ascii_letters+string.digits, k=30)),
+                    "string": "".join(random.choices(string.ascii_letters+string.digits, k=30)),
                     "dt": datetime.datetime.now()
                     }
                 )
         if len(result) % 10000 == 0:
             print("mongodb `test_mongo`.`example` inserted 10000 lines")
-
             collection.insert_many(result)
             result = []
 
 if __name__ == "__main__":
     init_mysql()
-    init_mongo()
+    # init_mongo()
 
diff --git a/sit/docker/clickhouse/config.xml b/sit/docker/clickhouse/config.xml
@@ -12,17 +12,17 @@
                     <port>9000</port>
                 </replica>
             </shard>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>ch-server-3</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>ch-server-4</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
+<!--            <shard>-->
+<!--                <internal_replication>true</internal_replication>-->
+<!--                <replica>-->
+<!--                    <host>ch-server-3</host>-->
+<!--                    <port>9000</port>-->
+<!--                </replica>-->
+<!--                <replica>-->
+<!--                    <host>ch-server-4</host>-->
+<!--                    <port>9000</port>-->
+<!--                </replica>-->
+<!--            </shard>-->
         </bip_ck_cluster>
     </remote_servers>
     <zookeeper>

diff --git a/sit/docker/clickhouse/docker-compose.yml b/sit/docker/clickhouse/docker-compose.yml
@@ -21,23 +21,23 @@ services:
       - 9001:9000
     environment:
       TZ: "Asia/Shanghai"
-  ch-server-3:
-    image: yandex/clickhouse-server
-    volumes:
-      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
-      - ./macros3.xml:/etc/clickhouse-server/config.d/macros.xml
-    ports:
-      - 8125:8123
-      - 9002:9000
-    environment:
-      TZ: "Asia/Shanghai"
-  ch-server-4:
-    image: yandex/clickhouse-server
-    volumes:
-      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
-      - ./macros4.xml:/etc/clickhouse-server/config.d/macros.xml
-    ports:
-      - 8126:8123
-      - 9003:9000
-    environment:
-      TZ: "Asia/Shanghai"
+#  ch-server-3:
+#    image: yandex/clickhouse-server
+#    volumes:
+#      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
+#      - ./macros3.xml:/etc/clickhouse-server/config.d/macros.xml
+#    ports:
+#      - 8125:8123
+#      - 9002:9000
+#    environment:
+#      TZ: "Asia/Shanghai"
+#  ch-server-4:
+#    image: yandex/clickhouse-server
+#    volumes:
+#      - ./config.xml:/etc/clickhouse-server/config.d/local.xml
+#      - ./macros4.xml:/etc/clickhouse-server/config.d/macros.xml
+#    ports:
+#      - 8126:8123
+#      - 9003:9000
+#    environment:
+#      TZ: "Asia/Shanghai"