From ddd0f190df85692bc307cacc28e921cf5b469794 Mon Sep 17 00:00:00 2001 From: LUA Date: Mon, 11 Jan 2021 18:24:29 +0800 Subject: [PATCH] feat: JSON mode configuration takes effect (#2403) * fix: merge commit feat: JSON mode configuration takes effect #2386 feat: table dble_config to view the json configuration in memory inner-453 fix: json to map add unit test/update copyright fix: 1 for Dereferenced variable may be null fix: Dereferenced variable may be null * fix: dble_information adds distributed locks --- .../java/com/actiontech/dble/DbleServer.java | 19 +- .../backend/datasource/PhysicalDbGroup.java | 2 +- .../dble/backend/datasource/ShardingNode.java | 9 + .../dble/cluster/ClusterHelper.java | 8 +- .../actiontech/dble/cluster/ClusterLogic.java | 331 +++++---- .../dble/cluster/ClusterPathUtil.java | 14 +- .../cluster/general/AbstractConsulSender.java | 9 +- .../cluster/general/kVtoXml/ClusterToXml.java | 9 +- .../response/SequencePropertiesLoader.java | 4 +- .../cluster/general/response/XmlDbLoader.java | 12 +- .../general/response/XmlShardingLoader.java | 21 +- .../general/response/XmlUserLoader.java | 19 +- .../cluster/general/xmltoKv/XmltoCluster.java | 9 +- .../dble/cluster/zkprocess/ZkSender.java | 9 +- .../zkprocess/entity/dbGroups/DBInstance.java | 16 +- .../zkprocess/entity/dbGroups/HeartBeat.java | 4 + .../zkprocess/xmltozk/XmltoZkMain.java | 11 +- .../xmltozk/listen/DbXmlToZkLoader.java | 10 +- .../xmltozk/listen/SequenceToZkLoader.java | 2 +- .../xmltozk/listen/ShardingXmlToZKLoader.java | 18 +- .../xmltozk/listen/UserXmlToZkLoader.java | 18 +- .../zkprocess/zktoxml/ZktoXmlMain.java | 11 +- .../listen/DbGroupsZKToXmlListener.java | 10 +- .../listen/SequenceToPropertiesListener.java | 2 +- .../listen/ShardingZkToXmlListener.java | 18 +- .../zktoxml/listen/UserZkToXmlListener.java | 20 +- .../dble/config/ConfigInitializer.java | 64 ++ .../dble/config/DbleTempConfig.java | 61 ++ .../actiontech/dble/config/ServerConfig.java | 114 ++- .../dble/config/converter/DBConverter.java | 244 ++++++ .../config/converter/SequenceConverter.java | 48 ++ .../config/converter/ShardingConverter.java | 700 ++++++++++++++++++ .../dble/config/converter/UserConverter.java | 303 ++++++++ .../dble/config/loader/xml/XMLDbLoader.java | 6 +- .../config/loader/xml/XMLShardingLoader.java | 6 +- .../dble/config/loader/xml/XMLUserLoader.java | 2 +- .../config/model/sharding/SchemaConfig.java | 19 + .../model/sharding/table/BaseTableConfig.java | 16 + .../sharding/table/ChildTableConfig.java | 20 + .../sharding/table/GlobalTableConfig.java | 10 + .../sharding/table/ShardingTableConfig.java | 12 +- .../config/model/user/ManagerUserConfig.java | 5 + .../config/model/user/RwSplitUserConfig.java | 6 + .../config/model/user/ServerUserConfig.java | 19 + .../config/model/user/ShardingUserConfig.java | 14 + .../dble/config/model/user/UserConfig.java | 8 + .../model/user/UserPrivilegesConfig.java | 23 + .../dble/config/util/DbXmlWriteJob.java | 3 + .../PartitionByJumpConsistentHash.java | 26 + .../handler/IncrSequenceMySQLHandler.java | 10 + .../handler/IncrSequenceZKHandler.java | 37 +- .../sequence/handler/SequenceHandler.java | 3 + .../dble/route/util/PropertiesUtil.java | 18 +- .../manager/handler/DeleteHandler.java | 33 +- .../manager/handler/InsertHandler.java | 34 +- .../manager/handler/UpdateHandler.java | 131 ++-- .../manager/information/ManagerBaseTable.java | 9 + .../information/ManagerSchemaInfo.java | 1 + .../information/ManagerWritableTable.java | 31 +- .../information/tables/DbleConfig.java | 71 ++ .../information/tables/DbleDbGroup.java | 117 ++- .../information/tables/DbleDbInstance.java | 394 ++++------ .../information/tables/DbleRwSplitEntry.java | 154 ++-- .../services/manager/response/DryRun.java | 20 +- .../manager/response/ReloadConfig.java | 94 ++- .../dble/singleton/SequenceManager.java | 4 + .../com/actiontech/dble/util/StringUtil.java | 4 + .../dble/cluster/ClusterHelpTest.java | 213 ++++-- 68 files changed, 2804 insertions(+), 918 deletions(-) create mode 100644 src/main/java/com/actiontech/dble/config/DbleTempConfig.java create mode 100644 src/main/java/com/actiontech/dble/config/converter/DBConverter.java create mode 100644 src/main/java/com/actiontech/dble/config/converter/SequenceConverter.java create mode 100644 src/main/java/com/actiontech/dble/config/converter/ShardingConverter.java create mode 100644 src/main/java/com/actiontech/dble/config/converter/UserConverter.java create mode 100644 src/main/java/com/actiontech/dble/services/manager/information/tables/DbleConfig.java diff --git a/src/main/java/com/actiontech/dble/DbleServer.java b/src/main/java/com/actiontech/dble/DbleServer.java index b0c5b89e05..9a0cdeb63d 100644 --- a/src/main/java/com/actiontech/dble/DbleServer.java +++ b/src/main/java/com/actiontech/dble/DbleServer.java @@ -13,6 +13,7 @@ import com.actiontech.dble.backend.mysql.xa.recovery.impl.FileSystemRepository; import com.actiontech.dble.backend.mysql.xa.recovery.impl.KVStoreRepository; import com.actiontech.dble.buffer.DirectByteBufferPool; +import com.actiontech.dble.config.DbleTempConfig; import com.actiontech.dble.config.ServerConfig; import com.actiontech.dble.config.model.ClusterConfig; import com.actiontech.dble.config.model.SystemConfig; @@ -113,7 +114,7 @@ private DbleServer() { public void startup() throws Exception { LOGGER.info("===========================================DBLE SERVER STARTING==================================="); - this.config = new ServerConfig(); + initServerConfig(); this.startupTime = TimeUtil.currentTimeMillis(); LOGGER.info("=========================================Config file read finish=================================="); @@ -258,10 +259,22 @@ public void startup() throws Exception { LOGGER.info("====================================CronScheduler started========================================="); CustomMySQLHa.getInstance().start(); + LOGGER.info("======================================ALL START INIT FINISH======================================="); startup = true; } + private void initServerConfig() throws Exception { + if (ClusterConfig.getInstance().isClusterEnable()) { + this.config = new ServerConfig(DbleTempConfig.getInstance().getUserConfig(), DbleTempConfig.getInstance().getDbConfig(), + DbleTempConfig.getInstance().getShardingConfig(), DbleTempConfig.getInstance().getSequenceConfig()); + DbleTempConfig.getInstance().clean(); + this.config.syncJsonToLocal(true); + } else { + this.config = new ServerConfig(); + } + } + private void initAioProcessor(int processorCount) throws IOException { for (int i = 0; i < processorCount; i++) { asyncChannelGroups[i] = AsynchronousChannelGroup.withFixedThreadPool(processorCount, @@ -386,10 +399,10 @@ public void run() { private void reviseSchemas() { if (systemVariables.isLowerCaseTableNames()) { - config.reviseLowerCase(); + config.reviseLowerCase(DbleTempConfig.getInstance().getSequenceConfig()); ConfigUtil.setSchemasForPool(config.getDbGroups(), config.getShardingNodes()); } else { - config.loadSequence(); + config.loadSequence(DbleTempConfig.getInstance().getSequenceConfig()); config.selfChecking0(); } } diff --git a/src/main/java/com/actiontech/dble/backend/datasource/PhysicalDbGroup.java b/src/main/java/com/actiontech/dble/backend/datasource/PhysicalDbGroup.java index 806ff0b2e2..445986bb8b 100644 --- a/src/main/java/com/actiontech/dble/backend/datasource/PhysicalDbGroup.java +++ b/src/main/java/com/actiontech/dble/backend/datasource/PhysicalDbGroup.java @@ -447,7 +447,7 @@ public boolean checkInstanceExist(String instanceName) { return true; } - boolean equalsBaseInfo(PhysicalDbGroup pool) { + public boolean equalsBaseInfo(PhysicalDbGroup pool) { return pool.getDbGroupConfig().getName().equals(this.dbGroupConfig.getName()) && pool.getDbGroupConfig().getHeartbeatSQL().equals(this.dbGroupConfig.getHeartbeatSQL()) && pool.getDbGroupConfig().getHeartbeatTimeout() == this.dbGroupConfig.getHeartbeatTimeout() && diff --git a/src/main/java/com/actiontech/dble/backend/datasource/ShardingNode.java b/src/main/java/com/actiontech/dble/backend/datasource/ShardingNode.java index 474ac52585..cedddfd014 100644 --- a/src/main/java/com/actiontech/dble/backend/datasource/ShardingNode.java +++ b/src/main/java/com/actiontech/dble/backend/datasource/ShardingNode.java @@ -9,6 +9,7 @@ import com.actiontech.dble.net.connection.BackendConnection; import com.actiontech.dble.route.RouteResultsetNode; import com.actiontech.dble.singleton.TraceManager; +import com.actiontech.dble.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -122,4 +123,12 @@ private Boolean canRunOnMaster(RouteResultsetNode rrs, boolean autoCommit) { } return master; } + + + public boolean equalsBaseInfo(ShardingNode shardingNode) { + return StringUtil.equalsWithEmpty(this.name, shardingNode.getName()) && + StringUtil.equalsWithEmpty(this.dbGroupName, shardingNode.getDbGroupName()) && + StringUtil.equalsWithEmpty(this.database, shardingNode.getDatabase()) && + this.dbGroup.equalsBaseInfo(shardingNode.getDbGroup()); + } } diff --git a/src/main/java/com/actiontech/dble/cluster/ClusterHelper.java b/src/main/java/com/actiontech/dble/cluster/ClusterHelper.java index b09c383e8b..777f16e025 100644 --- a/src/main/java/com/actiontech/dble/cluster/ClusterHelper.java +++ b/src/main/java/com/actiontech/dble/cluster/ClusterHelper.java @@ -23,6 +23,9 @@ public static void createSelfTempNode(String path, String value) throws Exceptio } public static KvBean getKV(String path) throws Exception { + if (ClusterGeneralConfig.getInstance().getClusterSender() == null) { + return null; + } return ClusterGeneralConfig.getInstance().getClusterSender().getKV(path); } @@ -52,7 +55,10 @@ public static Map getOnlineMap() { } public static void writeConfToCluster() throws Exception { - ClusterGeneralConfig.getInstance().getClusterSender().writeConfToCluster(); + ClusterLogic.syncSequenceJsonToCluster(); + ClusterLogic.syncDbJsonToCluster(); + ClusterLogic.syncShardingJsonToCluster(); + ClusterLogic.syncUseJsonToCluster(); } public static String getPathValue(String path) throws Exception { diff --git a/src/main/java/com/actiontech/dble/cluster/ClusterLogic.java b/src/main/java/com/actiontech/dble/cluster/ClusterLogic.java index c21dada2f4..7cd51955ce 100644 --- a/src/main/java/com/actiontech/dble/cluster/ClusterLogic.java +++ b/src/main/java/com/actiontech/dble/cluster/ClusterLogic.java @@ -27,6 +27,11 @@ import com.actiontech.dble.cluster.zkprocess.entity.user.User; import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.DbleTempConfig; +import com.actiontech.dble.config.converter.DBConverter; +import com.actiontech.dble.config.converter.SequenceConverter; +import com.actiontech.dble.config.converter.ShardingConverter; +import com.actiontech.dble.config.converter.UserConverter; import com.actiontech.dble.config.model.ClusterConfig; import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.meta.ReloadManager; @@ -35,6 +40,7 @@ import com.actiontech.dble.net.connection.BackendConnection; import com.actiontech.dble.net.connection.FrontendConnection; import com.actiontech.dble.route.RouteResultsetNode; +import com.actiontech.dble.route.util.PropertiesUtil; import com.actiontech.dble.services.manager.response.ReloadConfig; import com.actiontech.dble.services.manager.response.ShowBinlogStatus; import com.actiontech.dble.services.mysqlsharding.ShardingService; @@ -257,7 +263,7 @@ public static void reloadConfigEvent(String value, String params) throws Excepti return; } try { - boolean result = ReloadConfig.reloadAll(Integer.parseInt(params)); + boolean result = ReloadConfig.reloadByConfig(Integer.parseInt(params), false); if (!checkLocalResult(result)) { return; } @@ -389,43 +395,68 @@ public static void checkPauseStatusRelease(String crashNode) { } } - public static void syncSequenceToCluster() throws Exception { + /** + * sequence + * properties -> cluster + * + * @throws Exception + */ + public static void syncSequencePropsToCluster() throws Exception { if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_ZK_GLOBAL_INCREMENT) { - JsonObject jsonObject = new JsonObject(); - String sequenceConf = ConfFileRWUtils.readFile(ConfigFileName.SEQUENCE_FILE_NAME); - jsonObject.addProperty(ConfigFileName.SEQUENCE_FILE_NAME, sequenceConf); - ClusterHelper.setKV(ClusterPathUtil.getSequencesCommonPath(), (new Gson()).toJson(jsonObject)); + String json = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_FILE_NAME); + ClusterHelper.setKV(ClusterPathUtil.getSequencesCommonPath(), json); LOGGER.info("Sequence To cluster: " + ConfigFileName.SEQUENCE_FILE_NAME + " ,success"); } else if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_MYSQL) { - JsonObject jsonObject = new JsonObject(); - String sequenceDbConf = ConfFileRWUtils.readFile(ConfigFileName.SEQUENCE_DB_FILE_NAME); - jsonObject.addProperty(ConfigFileName.SEQUENCE_DB_FILE_NAME, sequenceDbConf); - ClusterHelper.setKV(ClusterPathUtil.getSequencesCommonPath(), (new Gson()).toJson(jsonObject)); + String json = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_DB_FILE_NAME); + ClusterHelper.setKV(ClusterPathUtil.getSequencesCommonPath(), json); LOGGER.info("Sequence To cluster: " + ConfigFileName.SEQUENCE_DB_FILE_NAME + " ,success"); } } - public static void syncSequenceToLocal(KvBean configValue) throws Exception { + /** + * sequence + * json -> cluster + * + * @throws Exception + */ + public static void syncSequenceJsonToCluster() throws Exception { + String sequenceConfig = DbleServer.getInstance().getConfig().getSequenceConfig(); + if (null == sequenceConfig) { + LOGGER.info("sequence config is null"); + return; + } + ClusterHelper.setKV(ClusterPathUtil.getSequencesCommonPath(), sequenceConfig); + LOGGER.info("Sequence To cluster: " + sequenceConfig + " ,success"); + } + + + public static void syncSequenceToLocal(String sequenceConfig, boolean isWriteToLocal) throws Exception { String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); - if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock) && !isWriteToLocal) { return; } - if (!StringUtil.isEmpty(configValue.getValue())) { - JsonObject jsonObj = new JsonParser().parse(configValue.getValue()).getAsJsonObject(); - if (jsonObj.get(ConfigFileName.SEQUENCE_FILE_NAME) != null) { - String sequenceConf = jsonObj.get(ConfigFileName.SEQUENCE_FILE_NAME).getAsString(); - ConfFileRWUtils.writeFile(ConfigFileName.SEQUENCE_FILE_NAME, sequenceConf); - LOGGER.info("Sequence To Local: " + ConfigFileName.SEQUENCE_FILE_NAME + " ,success"); - } else if (jsonObj.get(ConfigFileName.SEQUENCE_DB_FILE_NAME) != null) { - String sequenceConf = jsonObj.get(ConfigFileName.SEQUENCE_DB_FILE_NAME).getAsString(); - ConfFileRWUtils.writeFile(ConfigFileName.SEQUENCE_DB_FILE_NAME, sequenceConf); - LOGGER.info("Sequence To Local: " + ConfigFileName.SEQUENCE_DB_FILE_NAME + " ,success"); - } + if (!StringUtil.isEmpty(sequenceConfig)) { + SequenceConverter sequenceConverter = new SequenceConverter(); + Properties props = sequenceConverter.jsonToProperties(sequenceConfig); + PropertiesUtil.storeProps(props, sequenceConverter.getFileName()); + LOGGER.info("Sequence To Local: " + sequenceConverter.getFileName() + " ,success"); } else { - LOGGER.warn("Sequence To Local: get empty value,path is" + configValue.getKey()); + LOGGER.warn("Sequence To Local: get empty value"); } } + public static void syncSequenceJson(KvBean configValue) throws Exception { + LOGGER.info("start sync sequence json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + return; + } + + DbleTempConfig.getInstance().setSequenceConfig(configValue.getValue()); + + LOGGER.info("end sync sequence json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + } + private static void changeDbGroupByStatus(DBGroup dbGroup, List statusList) { Map statusMap = new HashMap<>(statusList.size()); for (DbInstanceStatus status : statusList) { @@ -490,73 +521,8 @@ static void writeMapFileAddFunction(List functionList) { } - static String parseShardingXmlFileToJson(XmlProcessBase xmlParseBase, Gson gson, String path) throws JAXBException, XMLStreamException { - // xml file to bean - Shardings shardingBean; - try { - shardingBean = (Shardings) xmlParseBase.baseParseXmlToBean(path); - } catch (Exception e) { - LOGGER.warn("parseXmlToBean Exception", e); - throw e; - } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Xml to Shardings is :" + shardingBean); - } - // bean to json obj - JsonObject jsonObj = new JsonObject(); - jsonObj.addProperty(ClusterPathUtil.VERSION, shardingBean.getVersion()); - - JsonArray schemaArray = new JsonArray(); - for (Schema schema : shardingBean.getSchema()) { - if (schema.getTable() != null) { - JsonObject schemaJsonObj = gson.toJsonTree(schema).getAsJsonObject(); - schemaJsonObj.remove("table"); - JsonArray tableArray = new JsonArray(); - for (Object table : schema.getTable()) { - JsonElement tableElement = gson.toJsonTree(table, Table.class); - tableArray.add(tableElement); - } - schemaJsonObj.add("table", gson.toJsonTree(tableArray)); - schemaArray.add(gson.toJsonTree(schemaJsonObj)); - } else { - schemaArray.add(gson.toJsonTree(schema)); - } - } - jsonObj.add(ClusterPathUtil.SCHEMA, gson.toJsonTree(schemaArray)); - jsonObj.add(ClusterPathUtil.SHARDING_NODE, gson.toJsonTree(shardingBean.getShardingNode())); - List functionList = shardingBean.getFunction(); - readMapFileAddFunction(functionList); - jsonObj.add(ClusterPathUtil.FUNCTION, gson.toJsonTree(functionList)); - //from json obj to string - return gson.toJson(jsonObj); - } - - private static void readMapFileAddFunction(List functionList) { - List tempData = new ArrayList<>(); - for (Function function : functionList) { - List proList = function.getProperty(); - if (null != proList && !proList.isEmpty()) { - for (Property property : proList) { - // if mapfile,read and save to json - if (ParseParamEnum.ZK_PATH_RULE_MAPFILE_NAME.getKey().equals(property.getName())) { - Property mapFilePro = new Property(); - mapFilePro.setName(property.getValue()); - try { - mapFilePro.setValue(ConfFileRWUtils.readFile(property.getValue())); - tempData.add(mapFilePro); - } catch (IOException e) { - LOGGER.warn("readMapFile IOException", e); - } - } - } - proList.addAll(tempData); - tempData.clear(); - } - } - } - - static Shardings parseShardingJsonToBean(Gson gson, String jsonContent) { + public static Shardings parseShardingJsonToBean(Gson gson, String jsonContent) { //from string to json obj JsonObject jsonObject = new JsonParser().parse(jsonContent).getAsJsonObject(); @@ -603,7 +569,7 @@ static Shardings parseShardingJsonToBean(Gson gson, String jsonContent) { return shardingBean; } - static DbGroups parseDbGroupsJsonToBean(Gson gson, String jsonContent) { + public static DbGroups parseDbGroupsJsonToBean(Gson gson, String jsonContent) { DbGroups dbs = new DbGroups(); JsonObject jsonObject = new JsonParser().parse(jsonContent).getAsJsonObject(); JsonElement dbGroupsJson = jsonObject.get(ClusterPathUtil.DB_GROUP); @@ -612,7 +578,9 @@ static DbGroups parseDbGroupsJsonToBean(Gson gson, String jsonContent) { new TypeToken>() { }.getType()); dbs.setDbGroup(dbGroupList); - syncHaStatusFromCluster(gson, dbs, dbGroupList); + if (ClusterConfig.getInstance().isClusterEnable()) { + syncHaStatusFromCluster(gson, dbs, dbGroupList); + } } JsonElement version = jsonObject.get(ClusterPathUtil.VERSION); @@ -677,35 +645,8 @@ static String parseDbGroupXmlFileToJson(XmlProcessBase xmlParseBase, Gson gson, return gson.toJson(jsonObj); } - static String parseUserXmlFileToJson(XmlProcessBase xmlParseBase, Gson gson, String path) throws JAXBException, XMLStreamException { - // xml file to bean - Users usersBean; - try { - usersBean = (Users) xmlParseBase.baseParseXmlToBean(path); - } catch (Exception e) { - LOGGER.warn("parseXmlToBean Exception", e); - throw e; - } - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Xml to Shardings is :" + usersBean); - } - // bean to json obj - JsonObject jsonObj = new JsonObject(); - jsonObj.addProperty(ClusterPathUtil.VERSION, usersBean.getVersion()); - - JsonArray userArray = new JsonArray(); - for (Object user : usersBean.getUser()) { - JsonElement tableElement = gson.toJsonTree(user, User.class); - userArray.add(tableElement); - } - jsonObj.add(ClusterPathUtil.USER, gson.toJsonTree(userArray)); - jsonObj.add(ClusterPathUtil.BLACKLIST, gson.toJsonTree(usersBean.getBlacklist())); - //from json obj to string - return gson.toJson(jsonObj); - } - static Users parseUserJsonToBean(Gson gson, String jsonContent) { + public static Users parseUserJsonToBean(Gson gson, String jsonContent) { //from string to json obj JsonObject jsonObject = new JsonParser().parse(jsonContent).getAsJsonObject(); @@ -830,13 +771,14 @@ public static String waitingForAllTheNode(String path, String checkString) throw } } - public static void syncDbXmlToLocal(XmlProcessBase xmlParseBase, KvBean configValue) throws Exception { - LOGGER.info("cluster to local " + ConfigFileName.DB_XML + " start:" + configValue.getKey() + " " + configValue.getValue()); + public static void syncDbXmlToLocal(XmlProcessBase xmlParseBase, String dbConfig, boolean isWriteToLocal) throws Exception { + LOGGER.info("cluster to local " + ConfigFileName.DB_XML + " start:" + dbConfig); String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); - if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock) && !isWriteToLocal) { return; } - DbGroups dbs = ClusterLogic.parseDbGroupsJsonToBean(new Gson(), configValue.getValue()); + + DbGroups dbs = ClusterLogic.parseDbGroupsJsonToBean(new Gson(), dbConfig); String path = ResourceUtil.getResourcePathFromRoot(ClusterPathUtil.LOCAL_WRITE_PATH); path = new File(path).getPath() + File.separator + ConfigFileName.DB_XML; @@ -848,31 +790,86 @@ public static void syncDbXmlToLocal(XmlProcessBase xmlParseBase, KvBean configVa LOGGER.info("cluster to local xml write :" + path + " is success"); } - public static void syncDbXmlToCluster(XmlProcessBase xmlParseBase) throws Exception { + public static void syncDbJson(KvBean configValue) throws Exception { + LOGGER.info("start sync db json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + return; + } + + DbleTempConfig.getInstance().setDbConfig(configValue.getValue()); + + LOGGER.info("end sync db json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + } + + /** + * db + * xml -> cluster + * + * @throws Exception + */ + public static void syncDbXmlToCluster() throws Exception { LOGGER.info(ConfigFileName.DB_XML + " local to cluster start"); - String path = ClusterPathUtil.LOCAL_WRITE_PATH + ConfigFileName.DB_XML; - String json = parseDbGroupXmlFileToJson(xmlParseBase, new Gson(), path); + String json = DBConverter.dbXmlToJson(); ClusterHelper.setKV(ClusterPathUtil.getDbConfPath(), json); - LOGGER.info("xml local to cluster write :" + path + " is success"); + LOGGER.info("xml local to cluster write is success"); + } + + /** + * db + * json -> cluster + * + * @throws Exception + */ + public static void syncDbJsonToCluster() throws Exception { + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + if (null == dbConfig) { + LOGGER.info("db config is null"); + return; + } + ClusterHelper.setKV(ClusterPathUtil.getDbConfPath(), dbConfig); + LOGGER.info("db json config to cluster write is success"); } - public static void syncShardingXmlToCluster(XmlProcessBase xmlParseBase, Gson gson) throws Exception { + /** + * sharding + * xml -> cluster + * + * @throws Exception + */ + public static void syncShardingXmlToCluster() throws Exception { LOGGER.info(ConfigFileName.SHARDING_XML + " local to cluster start"); - String path = ClusterPathUtil.LOCAL_WRITE_PATH + ConfigFileName.SHARDING_XML; - String json = ClusterLogic.parseShardingXmlFileToJson(xmlParseBase, gson, path); + ShardingConverter shardingConverter = new ShardingConverter(); + String json = shardingConverter.shardingXmlToJson(); ClusterHelper.setKV(ClusterPathUtil.getConfShardingPath(), json); - LOGGER.info("xml local to cluster write :" + path + " is success"); + LOGGER.info("xml local to cluster write is success"); } - public static void syncShardingXmlToLocal(KvBean configValue, XmlProcessBase xmlParseBase, Gson gson) throws Exception { - LOGGER.info("cluster to local " + ConfigFileName.SHARDING_XML + " start:" + configValue.getKey() + " " + configValue.getValue()); + /** + * sharding + * json -> cluster + * + * @throws Exception + */ + public static void syncShardingJsonToCluster() throws Exception { + String shardingConfig = DbleServer.getInstance().getConfig().getShardingConfig(); + if (null == shardingConfig) { + LOGGER.info("sharding config is null"); + return; + } + ClusterHelper.setKV(ClusterPathUtil.getConfShardingPath(), shardingConfig); + LOGGER.info("sharding json config to cluster write is success"); + } + + public static void syncShardingXmlToLocal(String shardingConfig, XmlProcessBase xmlParseBase, Gson gson, boolean isWriteToLocal) throws Exception { + LOGGER.info("cluster to local " + ConfigFileName.SHARDING_XML + " start:" + shardingConfig); String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); - if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock) && !isWriteToLocal) { return; } //the config Value in ucore is an all in one json config of the sharding.xml - Shardings sharding = ClusterLogic.parseShardingJsonToBean(gson, configValue.getValue()); + Shardings sharding = ClusterLogic.parseShardingJsonToBean(gson, shardingConfig); ClusterLogic.writeMapFileAddFunction(sharding.getFunction()); String path = ResourceUtil.getResourcePathFromRoot(ClusterPathUtil.LOCAL_WRITE_PATH); @@ -886,23 +883,57 @@ public static void syncShardingXmlToLocal(KvBean configValue, XmlProcessBase xml } - public static void syncUserXmlToCluster(XmlProcessBase xmlParseBase, Gson gson) throws Exception { + public static void syncShardingJson(KvBean configValue) throws Exception { + LOGGER.info("start sync sharding json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + return; + } + + DbleTempConfig.getInstance().setShardingConfig(configValue.getValue()); + + LOGGER.info("end sync sharding json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + } + + + /** + * user + * xml -> cluster + * + * @throws Exception + */ + public static void syncUserXmlToCluster() throws Exception { LOGGER.info(ConfigFileName.USER_XML + " local to cluster start"); - String path = ClusterPathUtil.LOCAL_WRITE_PATH + ConfigFileName.USER_XML; - String json = ClusterLogic.parseUserXmlFileToJson(xmlParseBase, gson, path); + String json = new UserConverter().userXmlToJson(); ClusterHelper.setKV(ClusterPathUtil.getUserConfPath(), json); - LOGGER.info("xml local to cluster write :" + path + " is success"); + LOGGER.info("xml local to cluster write is success"); } - public static void syncUserXmlToLocal(KvBean configValue, XmlProcessBase xmlParseBase, Gson gson) throws Exception { - LOGGER.info("cluster to local " + ConfigFileName.USER_XML + " start:" + configValue.getKey() + " " + configValue.getValue()); + /** + * user + * json -> cluster + * + * @throws Exception + */ + public static void syncUseJsonToCluster() throws Exception { + String userConfig = DbleServer.getInstance().getConfig().getUserConfig(); + if (null == userConfig) { + LOGGER.info("user config is null"); + return; + } + ClusterHelper.setKV(ClusterPathUtil.getUserConfPath(), userConfig); + LOGGER.info("user json config to cluster write is success"); + } + + public static void syncUserXmlToLocal(String userConfig, XmlProcessBase xmlParseBase, Gson gson, boolean isWriteToLocal) throws Exception { + LOGGER.info("cluster to local " + ConfigFileName.USER_XML + " start:" + userConfig); String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); - if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock) && !isWriteToLocal) { return; } //the config Value is an all in one json config of the user.xml - Users users = ClusterLogic.parseUserJsonToBean(gson, configValue.getValue()); + Users users = ClusterLogic.parseUserJsonToBean(gson, userConfig); String path = ResourceUtil.getResourcePathFromRoot(ClusterPathUtil.LOCAL_WRITE_PATH); path = new File(path).getPath() + File.separator + ConfigFileName.USER_XML; @@ -914,6 +945,18 @@ public static void syncUserXmlToLocal(KvBean configValue, XmlProcessBase xmlPars LOGGER.info("cluster to local write :" + path + " is success"); } + public static void syncUserJson(KvBean configValue) throws Exception { + LOGGER.info("start sync user json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + String lock = ClusterHelper.getPathValue(ClusterPathUtil.getConfChangeLockPath()); + if (lock != null && SystemConfig.getInstance().getInstanceName().equals(lock)) { + return; + } + + DbleTempConfig.getInstance().setUserConfig(configValue.getValue()); + + LOGGER.info("end sync user json config:key[{}],value[{}]", configValue.getKey(), configValue.getValue()); + } + public static void syncDbGroupStatusToCluster() throws Exception { LOGGER.info("syncDbGroupStatusToCluster start"); HaConfigManager.getInstance().init(); diff --git a/src/main/java/com/actiontech/dble/cluster/ClusterPathUtil.java b/src/main/java/com/actiontech/dble/cluster/ClusterPathUtil.java index fc7d135d17..87b97f0dd5 100644 --- a/src/main/java/com/actiontech/dble/cluster/ClusterPathUtil.java +++ b/src/main/java/com/actiontech/dble/cluster/ClusterPathUtil.java @@ -27,13 +27,13 @@ private ClusterPathUtil() { public static final String CONF_BASE_PATH = BASE_PATH + "conf" + SEPARATOR; //xml properties - static final String SCHEMA = "schema"; - static final String DB_GROUP = "dbGroup"; - static final String SHARDING_NODE = "shardingNode"; - static final String BLACKLIST = "blacklist"; - static final String VERSION = "version"; - static final String FUNCTION = "function"; - static final String USER = "user"; + public static final String SCHEMA = "schema"; + public static final String DB_GROUP = "dbGroup"; + public static final String SHARDING_NODE = "shardingNode"; + public static final String BLACKLIST = "blacklist"; + public static final String VERSION = "version"; + public static final String FUNCTION = "function"; + public static final String USER = "user"; public static final String SUCCESS = "success"; diff --git a/src/main/java/com/actiontech/dble/cluster/general/AbstractConsulSender.java b/src/main/java/com/actiontech/dble/cluster/general/AbstractConsulSender.java index ee8d5dad47..68fbd03bc0 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/AbstractConsulSender.java +++ b/src/main/java/com/actiontech/dble/cluster/general/AbstractConsulSender.java @@ -17,7 +17,6 @@ import com.actiontech.dble.cluster.general.kVtoXml.ClusterToXml; import com.actiontech.dble.cluster.general.listener.ClusterClearKeyListener; import com.actiontech.dble.cluster.general.response.*; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.singleton.OnlineStatus; import com.actiontech.dble.singleton.ProxyMeta; @@ -102,13 +101,11 @@ public void createSelfTempNode(String path, String value) throws Exception { public void writeConfToCluster() throws Exception { ClusterClearKeyListener ucoreListen = new ClusterClearKeyListener(this); - XmlProcessBase xmlProcess = new XmlProcessBase(); - new XmlDbLoader(xmlProcess, ucoreListen); - new XmlShardingLoader(xmlProcess, ucoreListen); - new XmlUserLoader(xmlProcess, ucoreListen); + new XmlDbLoader(ucoreListen); + new XmlShardingLoader(ucoreListen); + new XmlUserLoader(ucoreListen); new SequencePropertiesLoader(ucoreListen); - xmlProcess.initJaxbClass(); ucoreListen.initAllNode(); new DbGroupHaResponse().notifyCluster(); } diff --git a/src/main/java/com/actiontech/dble/cluster/general/kVtoXml/ClusterToXml.java b/src/main/java/com/actiontech/dble/cluster/general/kVtoXml/ClusterToXml.java index 8a94703a27..562ae5abd1 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/kVtoXml/ClusterToXml.java +++ b/src/main/java/com/actiontech/dble/cluster/general/kVtoXml/ClusterToXml.java @@ -11,7 +11,6 @@ import com.actiontech.dble.cluster.general.listener.ClusterOffLineListener; import com.actiontech.dble.cluster.general.listener.ClusterSingleKeyListener; import com.actiontech.dble.cluster.general.response.*; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.config.model.ClusterConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,13 +40,11 @@ public static void loadKVtoFile(AbstractConsulSender sender) { try { //create a new listener to the ucore config change listener = new ClusterClearKeyListener(sender); - XmlProcessBase xmlProcess = new XmlProcessBase(); //add all loader into listener map list - new XmlDbLoader(xmlProcess, listener); - new XmlShardingLoader(xmlProcess, listener); - new XmlUserLoader(xmlProcess, listener); + new XmlDbLoader(listener); + new XmlShardingLoader(listener); + new XmlUserLoader(listener); new SequencePropertiesLoader(listener); - xmlProcess.initJaxbClass(); //add listener to watch the Prefix of the keys new ConfigStatusResponse(listener); diff --git a/src/main/java/com/actiontech/dble/cluster/general/response/SequencePropertiesLoader.java b/src/main/java/com/actiontech/dble/cluster/general/response/SequencePropertiesLoader.java index 3d552948d4..a730b6b8f9 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/response/SequencePropertiesLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/general/response/SequencePropertiesLoader.java @@ -28,12 +28,12 @@ public SequencePropertiesLoader(ClusterClearKeyListener confListener) { @Override public void notifyProcess(KvBean configValue) throws Exception { LOGGER.info("notify " + configValue.getKey() + " " + configValue.getValue() + " " + configValue.getChangeType()); - ClusterLogic.syncSequenceToLocal(configValue); + ClusterLogic.syncSequenceJson(configValue); } @Override public void notifyCluster() throws Exception { - ClusterLogic.syncSequenceToCluster(); + ClusterLogic.syncSequencePropsToCluster(); } } diff --git a/src/main/java/com/actiontech/dble/cluster/general/response/XmlDbLoader.java b/src/main/java/com/actiontech/dble/cluster/general/response/XmlDbLoader.java index 5dad2a6528..21761e8780 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/response/XmlDbLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/general/response/XmlDbLoader.java @@ -9,8 +9,6 @@ import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.general.listener.ClusterClearKeyListener; -import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,25 +18,21 @@ public class XmlDbLoader implements ClusterXmlLoader { private static final Logger LOGGER = LoggerFactory.getLogger(XmlDbLoader.class); - private XmlProcessBase xmlParseBase; - - public XmlDbLoader(XmlProcessBase xmlParseBase, ClusterClearKeyListener confListener) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(DbGroups.class); + public XmlDbLoader(ClusterClearKeyListener confListener) { confListener.addChild(this, ClusterPathUtil.getDbConfPath()); } @Override public void notifyProcess(KvBean configValue) throws Exception { LOGGER.info("notify " + configValue.getKey() + " " + configValue.getValue() + " " + configValue.getChangeType()); - ClusterLogic.syncDbXmlToLocal(xmlParseBase, configValue); + ClusterLogic.syncDbJson(configValue); } @Override public void notifyCluster() throws Exception { - ClusterLogic.syncDbXmlToCluster(xmlParseBase); + ClusterLogic.syncDbXmlToCluster(); } } diff --git a/src/main/java/com/actiontech/dble/cluster/general/response/XmlShardingLoader.java b/src/main/java/com/actiontech/dble/cluster/general/response/XmlShardingLoader.java index c547b90e6c..56d9d824cc 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/response/XmlShardingLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/general/response/XmlShardingLoader.java @@ -9,12 +9,6 @@ import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.general.listener.ClusterClearKeyListener; -import com.actiontech.dble.cluster.zkprocess.entity.Shardings; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.Table; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.TableGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,26 +17,19 @@ */ public class XmlShardingLoader implements ClusterXmlLoader { private static final Logger LOGGER = LoggerFactory.getLogger(XmlShardingLoader.class); - private final Gson gson; - private XmlProcessBase xmlParseBase; - - public XmlShardingLoader(XmlProcessBase xmlParseBase, ClusterClearKeyListener confListener) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(Shardings.class); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Table.class, new TableGsonAdapter()); - gson = gsonBuilder.create(); + + public XmlShardingLoader(ClusterClearKeyListener confListener) { confListener.addChild(this, ClusterPathUtil.getConfShardingPath()); } @Override public void notifyProcess(KvBean configValue) throws Exception { - ClusterLogic.syncShardingXmlToLocal(configValue, xmlParseBase, gson); + ClusterLogic.syncShardingJson(configValue); } @Override public void notifyCluster() throws Exception { - ClusterLogic.syncShardingXmlToCluster(xmlParseBase, gson); + ClusterLogic.syncShardingXmlToCluster(); } diff --git a/src/main/java/com/actiontech/dble/cluster/general/response/XmlUserLoader.java b/src/main/java/com/actiontech/dble/cluster/general/response/XmlUserLoader.java index dd6de164a5..3b064776a1 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/response/XmlUserLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/general/response/XmlUserLoader.java @@ -9,33 +9,20 @@ import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.general.listener.ClusterClearKeyListener; -import com.actiontech.dble.cluster.zkprocess.entity.Users; -import com.actiontech.dble.cluster.zkprocess.entity.user.User; -import com.actiontech.dble.cluster.zkprocess.entity.user.UserGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; public class XmlUserLoader implements ClusterXmlLoader { - private final Gson gson; - private XmlProcessBase xmlParseBase; - public XmlUserLoader(XmlProcessBase xmlParseBase, ClusterClearKeyListener confListener) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(Users.class); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(User.class, new UserGsonAdapter()); - gson = gsonBuilder.create(); + public XmlUserLoader(ClusterClearKeyListener confListener) { confListener.addChild(this, ClusterPathUtil.getUserConfPath()); } @Override public void notifyProcess(KvBean configValue) throws Exception { - ClusterLogic.syncUserXmlToLocal(configValue, xmlParseBase, gson); + ClusterLogic.syncUserJson(configValue); } @Override public void notifyCluster() throws Exception { - ClusterLogic.syncUserXmlToCluster(xmlParseBase, gson); + ClusterLogic.syncUserXmlToCluster(); } } diff --git a/src/main/java/com/actiontech/dble/cluster/general/xmltoKv/XmltoCluster.java b/src/main/java/com/actiontech/dble/cluster/general/xmltoKv/XmltoCluster.java index 5a85d97a3f..df82b42f02 100644 --- a/src/main/java/com/actiontech/dble/cluster/general/xmltoKv/XmltoCluster.java +++ b/src/main/java/com/actiontech/dble/cluster/general/xmltoKv/XmltoCluster.java @@ -10,7 +10,6 @@ import com.actiontech.dble.cluster.general.AbstractConsulSender; import com.actiontech.dble.cluster.general.listener.ClusterClearKeyListener; import com.actiontech.dble.cluster.general.response.*; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; /** @@ -40,17 +39,15 @@ public static void main(String[] args) { private static void initFileToUcore(AbstractConsulSender sender) throws Exception { ClusterClearKeyListener ucoreListen = new ClusterClearKeyListener(sender); - XmlProcessBase xmlProcess = new XmlProcessBase(); - new XmlDbLoader(xmlProcess, ucoreListen); + new XmlDbLoader(ucoreListen); - new XmlShardingLoader(xmlProcess, ucoreListen); + new XmlShardingLoader(ucoreListen); - new XmlUserLoader(xmlProcess, ucoreListen); + new XmlUserLoader(ucoreListen); new SequencePropertiesLoader(ucoreListen); - xmlProcess.initJaxbClass(); ucoreListen.initAllNode(); new DbGroupHaResponse().notifyCluster(); } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/ZkSender.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/ZkSender.java index f4e18298c9..86c11f0bd4 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/ZkSender.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/ZkSender.java @@ -10,7 +10,6 @@ import com.actiontech.dble.cluster.DistributeLock; import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.cluster.zkprocess.xmltozk.listen.*; import com.actiontech.dble.cluster.zkprocess.zktoxml.ZktoXmlMain; import com.actiontech.dble.config.model.SystemConfig; @@ -114,19 +113,17 @@ public void createSelfTempNode(String path, String value) throws Exception { @Override public void writeConfToCluster() throws Exception { ZookeeperProcessListen zkListen = new ZookeeperProcessListen(); - XmlProcessBase xmlProcess = new XmlProcessBase(); // xmltozk for sharding - new ShardingXmlToZKLoader(zkListen, xmlProcess); + new ShardingXmlToZKLoader(zkListen); // xmltozk for db - new DbXmlToZkLoader(zkListen, xmlProcess); + new DbXmlToZkLoader(zkListen); // xmltozk for user - new UserXmlToZkLoader(zkListen, xmlProcess); + new UserXmlToZkLoader(zkListen); new SequenceToZkLoader(zkListen); new DbGroupStatusToZkLoader(zkListen); - xmlProcess.initJaxbClass(); zkListen.initAllNode(); zkListen.clearInited(); } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/DBInstance.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/DBInstance.java index 192722bc4a..f9ec1ab0cd 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/DBInstance.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/DBInstance.java @@ -8,10 +8,7 @@ import com.actiontech.dble.cluster.zkprocess.entity.Propertied; import com.actiontech.dble.cluster.zkprocess.entity.Property; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlType; +import javax.xml.bind.annotation.*; import java.util.ArrayList; import java.util.List; @@ -45,6 +42,9 @@ public class DBInstance implements Propertied { protected List property; + @XmlTransient + protected String dbGroup; + public DBInstance() { } @@ -172,6 +172,14 @@ public void setProperty(List property) { this.property = property; } + public void setDbGroup(String dbGroup) { + this.dbGroup = dbGroup; + } + + public String getDbGroup() { + return dbGroup; + } + @Override public String toString() { return "dbInstance [name=" + diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/HeartBeat.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/HeartBeat.java index 184d627368..99093b6412 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/HeartBeat.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/entity/dbGroups/HeartBeat.java @@ -46,6 +46,10 @@ public String getValue() { return value; } + public void setValue(String value) { + this.value = value; + } + @Override public String toString() { return "heartbeat [timeout=" + diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/XmltoZkMain.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/XmltoZkMain.java index 3ae4760b40..085289a2a3 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/XmltoZkMain.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/XmltoZkMain.java @@ -7,7 +7,6 @@ import com.actiontech.dble.cluster.ClusterController; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.cluster.zkprocess.xmltozk.listen.*; import com.actiontech.dble.config.model.ClusterConfig; import com.actiontech.dble.util.KVPathUtil; @@ -33,13 +32,11 @@ public static void main(String[] args) { public static void initFileToZK() throws Exception { ZookeeperProcessListen zkListen = new ZookeeperProcessListen(); - XmlProcessBase xmlProcess = new XmlProcessBase(); + new DbXmlToZkLoader(zkListen); - new DbXmlToZkLoader(zkListen, xmlProcess); + new ShardingXmlToZKLoader(zkListen); - new ShardingXmlToZKLoader(zkListen, xmlProcess); - - new UserXmlToZkLoader(zkListen, xmlProcess); + new UserXmlToZkLoader(zkListen); new SequenceToZkLoader(zkListen); @@ -47,8 +44,6 @@ public static void initFileToZK() throws Exception { new DbGroupStatusToZkLoader(zkListen); - xmlProcess.initJaxbClass(); - zkListen.initAllNode(); zkListen.clearInited(); String confInited = KVPathUtil.getConfInitedPath(); diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/DbXmlToZkLoader.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/DbXmlToZkLoader.java index 5575013f7d..37fd04687c 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/DbXmlToZkLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/DbXmlToZkLoader.java @@ -9,22 +9,16 @@ import com.actiontech.dble.cluster.ClusterLogic; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; public class DbXmlToZkLoader implements NotifyService { - private XmlProcessBase xmlParseBase; - public DbXmlToZkLoader(ZookeeperProcessListen zookeeperListen, - XmlProcessBase xmlParseBase) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(DbGroups.class); + public DbXmlToZkLoader(ZookeeperProcessListen zookeeperListen) { zookeeperListen.addToInit(this); } @Override public boolean notifyProcess() throws Exception { - ClusterLogic.syncDbXmlToCluster(xmlParseBase); + ClusterLogic.syncDbXmlToCluster(); return true; } } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/SequenceToZkLoader.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/SequenceToZkLoader.java index 3f8ccfc350..cb8116f3fe 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/SequenceToZkLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/SequenceToZkLoader.java @@ -16,7 +16,7 @@ public SequenceToZkLoader(ZookeeperProcessListen zookeeperListen) { @Override public boolean notifyProcess() throws Exception { - ClusterLogic.syncSequenceToCluster(); + ClusterLogic.syncSequencePropsToCluster(); return true; } } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/ShardingXmlToZKLoader.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/ShardingXmlToZKLoader.java index 8ff16b9b8a..e3ae624836 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/ShardingXmlToZKLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/ShardingXmlToZKLoader.java @@ -8,29 +8,15 @@ import com.actiontech.dble.cluster.ClusterLogic; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.entity.Shardings; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.Table; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.TableGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; public class ShardingXmlToZKLoader implements NotifyService { - private final Gson gson; - private XmlProcessBase xmlParseBase; - public ShardingXmlToZKLoader(ZookeeperProcessListen zookeeperListen, - XmlProcessBase xmlParseBase) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(Shardings.class); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Table.class, new TableGsonAdapter()); - gson = gsonBuilder.create(); + public ShardingXmlToZKLoader(ZookeeperProcessListen zookeeperListen) { zookeeperListen.addToInit(this); } @Override public boolean notifyProcess() throws Exception { - ClusterLogic.syncShardingXmlToCluster(xmlParseBase, gson); + ClusterLogic.syncShardingXmlToCluster(); return true; } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/UserXmlToZkLoader.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/UserXmlToZkLoader.java index 94429c3e6d..a32ec8a69d 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/UserXmlToZkLoader.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/xmltozk/listen/UserXmlToZkLoader.java @@ -9,30 +9,16 @@ import com.actiontech.dble.cluster.ClusterLogic; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.entity.Users; -import com.actiontech.dble.cluster.zkprocess.entity.user.User; -import com.actiontech.dble.cluster.zkprocess.entity.user.UserGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; public class UserXmlToZkLoader implements NotifyService { - private final Gson gson; - private XmlProcessBase xmlParseBase; - public UserXmlToZkLoader(ZookeeperProcessListen zookeeperListen, - XmlProcessBase xmlParseBase) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(Users.class); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(User.class, new UserGsonAdapter()); - gson = gsonBuilder.create(); + public UserXmlToZkLoader(ZookeeperProcessListen zookeeperListen) { zookeeperListen.addToInit(this); } @Override public boolean notifyProcess() throws Exception { - ClusterLogic.syncUserXmlToCluster(xmlParseBase, gson); + ClusterLogic.syncUserXmlToCluster(); return true; } } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/ZktoXmlMain.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/ZktoXmlMain.java index 06007d0861..6a454a5d17 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/ZktoXmlMain.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/ZktoXmlMain.java @@ -8,7 +8,6 @@ import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.cluster.zkprocess.xmltozk.XmltoZkMain; import com.actiontech.dble.cluster.zkprocess.zktoxml.listen.*; import com.actiontech.dble.util.KVPathUtil; @@ -54,16 +53,15 @@ private static void initListenerFromZK() throws Exception { LOGGER.info("initListenerFromZK start"); ZookeeperProcessListen zkListen = new ZookeeperProcessListen(); Set childService = new HashSet<>(); - XmlProcessBase xmlProcess = new XmlProcessBase(); // load sharding - childService.add(new ShardingZkToXmlListener(zkListen, xmlProcess)); + childService.add(new ShardingZkToXmlListener(zkListen)); // load db - childService.add(new DbGroupsZKToXmlListener(zkListen, xmlProcess)); + childService.add(new DbGroupsZKToXmlListener(zkListen)); // load user - childService.add(new UserZkToXmlListener(zkListen, xmlProcess)); + childService.add(new UserZkToXmlListener(zkListen)); // load sequence childService.add(new SequenceToPropertiesListener(zkListen)); @@ -78,9 +76,6 @@ private static void initListenerFromZK() throws Exception { ZKUtils.addChildPathCache(ClusterPathUtil.getPauseShardingNodePath(), pauseShardingNodeListener); - // init xml - xmlProcess.initJaxbClass(); - // notify all zkListen.initAllNode(); zkListen.clearInited(); diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/DbGroupsZKToXmlListener.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/DbGroupsZKToXmlListener.java index 49108c4056..3dfc2a289e 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/DbGroupsZKToXmlListener.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/DbGroupsZKToXmlListener.java @@ -11,17 +11,11 @@ import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; public class DbGroupsZKToXmlListener implements NotifyService { - private XmlProcessBase xmlParseBase; - public DbGroupsZKToXmlListener(ZookeeperProcessListen zookeeperListen, - XmlProcessBase xmlParseBase) { + public DbGroupsZKToXmlListener(ZookeeperProcessListen zookeeperListen) { zookeeperListen.addToInit(this); - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(DbGroups.class); } @Override @@ -30,7 +24,7 @@ public boolean notifyProcess() throws Exception { if (configValue == null) { throw new RuntimeException(ClusterPathUtil.getDbConfPath() + " is null"); } - ClusterLogic.syncDbXmlToLocal(xmlParseBase, configValue); + ClusterLogic.syncDbJson(configValue); return true; } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/SequenceToPropertiesListener.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/SequenceToPropertiesListener.java index f20fce6073..bfb70632dc 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/SequenceToPropertiesListener.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/SequenceToPropertiesListener.java @@ -31,7 +31,7 @@ public boolean notifyProcess() throws Exception { throw new RuntimeException(ClusterPathUtil.getSequencesCommonPath() + " is null"); } LOGGER.info("notify " + configValue.getKey() + " " + configValue.getValue()); - ClusterLogic.syncSequenceToLocal(configValue); + ClusterLogic.syncSequenceJson(configValue); return true; } } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/ShardingZkToXmlListener.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/ShardingZkToXmlListener.java index 70f4cd55bd..4987953db9 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/ShardingZkToXmlListener.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/ShardingZkToXmlListener.java @@ -11,24 +11,10 @@ import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.entity.Shardings; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.Table; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.TableGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; public class ShardingZkToXmlListener implements NotifyService { - private final Gson gson; - private XmlProcessBase xmlParseBase; - public ShardingZkToXmlListener(ZookeeperProcessListen zookeeperListen, - XmlProcessBase xmlParseBase) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(Shardings.class); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Table.class, new TableGsonAdapter()); - gson = gsonBuilder.create(); + public ShardingZkToXmlListener(ZookeeperProcessListen zookeeperListen) { zookeeperListen.addToInit(this); } @@ -38,7 +24,7 @@ public boolean notifyProcess() throws Exception { if (configValue == null) { throw new RuntimeException(ClusterPathUtil.getConfShardingPath() + " is null"); } - ClusterLogic.syncShardingXmlToLocal(configValue, xmlParseBase, gson); + ClusterLogic.syncShardingJson(configValue); return true; } diff --git a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/UserZkToXmlListener.java b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/UserZkToXmlListener.java index 38f547ecfb..01683ec01a 100644 --- a/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/UserZkToXmlListener.java +++ b/src/main/java/com/actiontech/dble/cluster/zkprocess/zktoxml/listen/UserZkToXmlListener.java @@ -11,24 +11,10 @@ import com.actiontech.dble.cluster.general.bean.KvBean; import com.actiontech.dble.cluster.zkprocess.comm.NotifyService; import com.actiontech.dble.cluster.zkprocess.comm.ZookeeperProcessListen; -import com.actiontech.dble.cluster.zkprocess.entity.Users; -import com.actiontech.dble.cluster.zkprocess.entity.user.User; -import com.actiontech.dble.cluster.zkprocess.entity.user.UserGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; public class UserZkToXmlListener implements NotifyService { - private final Gson gson; - private XmlProcessBase xmlParseBase; - - public UserZkToXmlListener(ZookeeperProcessListen zookeeperListen, - XmlProcessBase xmlParseBase) { - this.xmlParseBase = xmlParseBase; - xmlParseBase.addParseClass(Users.class); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(User.class, new UserGsonAdapter()); - gson = gsonBuilder.create(); + + public UserZkToXmlListener(ZookeeperProcessListen zookeeperListen) { zookeeperListen.addToInit(this); } @@ -38,7 +24,7 @@ public boolean notifyProcess() throws Exception { if (configValue == null) { throw new RuntimeException(ClusterPathUtil.getUserConfPath() + " is null"); } - ClusterLogic.syncUserXmlToLocal(configValue, xmlParseBase, gson); + ClusterLogic.syncUserJson(configValue); return true; } diff --git a/src/main/java/com/actiontech/dble/config/ConfigInitializer.java b/src/main/java/com/actiontech/dble/config/ConfigInitializer.java index ee3a462207..c874637280 100644 --- a/src/main/java/com/actiontech/dble/config/ConfigInitializer.java +++ b/src/main/java/com/actiontech/dble/config/ConfigInitializer.java @@ -8,6 +8,10 @@ import com.actiontech.dble.backend.datasource.PhysicalDbGroup; import com.actiontech.dble.backend.datasource.PhysicalDbInstance; import com.actiontech.dble.backend.datasource.ShardingNode; +import com.actiontech.dble.config.converter.DBConverter; +import com.actiontech.dble.config.converter.SequenceConverter; +import com.actiontech.dble.config.converter.ShardingConverter; +import com.actiontech.dble.config.converter.UserConverter; import com.actiontech.dble.config.helper.TestSchemasTask; import com.actiontech.dble.config.helper.TestTask; import com.actiontech.dble.config.loader.xml.XMLDbLoader; @@ -30,6 +34,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLStreamException; import java.util.*; /** @@ -47,6 +53,10 @@ public class ConfigInitializer implements ProblemReporter { private volatile boolean fullyConfigured = false; private volatile Map blacklistConfig; private volatile Map functions; + private String dbConfig; + private String shardingConfig; + private String userConfig; + private String sequenceConfig; private List errorInfos = new ArrayList<>(); @@ -78,13 +88,52 @@ public ConfigInitializer(boolean lowerCaseNames) { this.functions = Collections.EMPTY_MAP; } + //sync json + this.userConfig = new UserConverter().userXmlToJson(); + this.dbConfig = DBConverter.dbXmlToJson(); + if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_ZK_GLOBAL_INCREMENT) { + this.sequenceConfig = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_FILE_NAME); + } else if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_MYSQL) { + this.sequenceConfig = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_DB_FILE_NAME); + } + this.shardingConfig = new ShardingConverter().shardingXmlToJson(); checkRwSplitDbGroup(); checkWriteDbInstance(); + } catch (JAXBException | XMLStreamException e) { + throw new ConfigException(e); } finally { TraceManager.finishSpan(traceObject); } } + public ConfigInitializer(String userConfig, String dbConfig, String shardingConfig, String sequenceConfig) { + //user + UserConverter userConverter = new UserConverter(); + userConverter.userJsonToMap(userConfig, this); + this.users = userConverter.getUserConfigMap(); + this.blacklistConfig = userConverter.getBlackListConfigMap(); + this.userConfig = userConfig; + + //db + DBConverter dbConverter = new DBConverter(); + dbConverter.dbJsonToMap(dbConfig, this); + this.dbGroups = dbConverter.getDbGroupMap(); + this.dbConfig = dbConfig; + + //sharding + ShardingConverter shardingConverter = new ShardingConverter(); + shardingConverter.shardingJsonToMap(shardingConfig, dbConverter.getDbGroupMap(), sequenceConfig, this); + this.schemas = shardingConverter.getSchemaConfigMap(); + this.erRelations = shardingConverter.getErRelations(); + this.shardingNodes = shardingConverter.getShardingNodeMap(); + this.functions = shardingConverter.getFunctionMap(); + this.shardingConfig = shardingConfig; + this.sequenceConfig = sequenceConfig; + + checkRwSplitDbGroup(); + checkWriteDbInstance(); + } + @Override public void warn(String problem) { this.errorInfos.add(new ErrorInfo("Xml", "WARNING", problem)); @@ -344,4 +393,19 @@ public List getErrorInfos() { return errorInfos; } + public String getDbConfig() { + return dbConfig; + } + + public String getShardingConfig() { + return shardingConfig; + } + + public String getUserConfig() { + return userConfig; + } + + public String getSequenceConfig() { + return sequenceConfig; + } } diff --git a/src/main/java/com/actiontech/dble/config/DbleTempConfig.java b/src/main/java/com/actiontech/dble/config/DbleTempConfig.java new file mode 100644 index 0000000000..e0cace628f --- /dev/null +++ b/src/main/java/com/actiontech/dble/config/DbleTempConfig.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2016-2021 ActionTech. + * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher. + */ +package com.actiontech.dble.config; + +public final class DbleTempConfig { + + private static final DbleTempConfig INSTANCE = new DbleTempConfig(); + + private DbleTempConfig() { + } + + private String dbConfig; + private String shardingConfig; + private String userConfig; + private String sequenceConfig; + + public String getDbConfig() { + return dbConfig; + } + + public String getShardingConfig() { + return shardingConfig; + } + + public String getUserConfig() { + return userConfig; + } + + public String getSequenceConfig() { + return sequenceConfig; + } + + public void setDbConfig(String dbConfig) { + this.dbConfig = dbConfig; + } + + public void setShardingConfig(String shardingConfig) { + this.shardingConfig = shardingConfig; + } + + public void setUserConfig(String userConfig) { + this.userConfig = userConfig; + } + + public void setSequenceConfig(String sequenceConfig) { + this.sequenceConfig = sequenceConfig; + } + + public static DbleTempConfig getInstance() { + return INSTANCE; + } + + public void clean() { + this.dbConfig = null; + this.shardingConfig = null; + this.userConfig = null; + this.sequenceConfig = null; + } +} diff --git a/src/main/java/com/actiontech/dble/config/ServerConfig.java b/src/main/java/com/actiontech/dble/config/ServerConfig.java index e0b662a962..99e9a274d6 100644 --- a/src/main/java/com/actiontech/dble/config/ServerConfig.java +++ b/src/main/java/com/actiontech/dble/config/ServerConfig.java @@ -12,6 +12,15 @@ import com.actiontech.dble.backend.datasource.PhysicalDbGroup; import com.actiontech.dble.backend.datasource.PhysicalDbGroupDiff; import com.actiontech.dble.backend.datasource.ShardingNode; +import com.actiontech.dble.cluster.ClusterLogic; +import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; +import com.actiontech.dble.cluster.zkprocess.entity.Shardings; +import com.actiontech.dble.cluster.zkprocess.entity.Users; +import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.Table; +import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.TableGsonAdapter; +import com.actiontech.dble.cluster.zkprocess.entity.user.User; +import com.actiontech.dble.cluster.zkprocess.entity.user.UserGsonAdapter; +import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.config.model.sharding.SchemaConfig; import com.actiontech.dble.config.model.sharding.table.BaseTableConfig; import com.actiontech.dble.config.model.sharding.table.ERTable; @@ -32,6 +41,8 @@ import com.actiontech.dble.singleton.SequenceManager; import com.actiontech.dble.util.StringUtil; import com.actiontech.dble.util.TimeUtil; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +71,10 @@ public class ServerConfig { private ConfigInitializer confInitNew; private volatile Map blacklistConfig; private volatile Map functions; + private String dbConfig; + private String shardingConfig; + private String userConfig; + private String sequenceConfig; public ServerConfig() { //read sharding.xml,db.xml and user.xml @@ -78,6 +93,10 @@ public ServerConfig() { this.lock = new ReentrantReadWriteLock(); this.blacklistConfig = confInitNew.getBlacklistConfig(); + this.userConfig = confInitNew.getUserConfig(); + this.dbConfig = confInitNew.getDbConfig(); + this.shardingConfig = confInitNew.getShardingConfig(); + this.sequenceConfig = confInitNew.getSequenceConfig(); } @@ -98,6 +117,28 @@ public ServerConfig(ConfigInitializer confInit) { this.blacklistConfig = confInit.getBlacklistConfig(); } + public ServerConfig(String userConfig, String dbConfig, String shardingConfig, String sequenceConfig) { + confInitNew = new ConfigInitializer(userConfig, dbConfig, shardingConfig, sequenceConfig); + this.users = confInitNew.getUsers(); + this.dbGroups = confInitNew.getDbGroups(); + + this.schemas = confInitNew.getSchemas(); + this.shardingNodes = confInitNew.getShardingNodes(); + this.erRelations = confInitNew.getErRelations(); + this.functions = confInitNew.getFunctions(); + this.fullyConfigured = confInitNew.isFullyConfigured(); + ConfigUtil.setSchemasForPool(dbGroups, shardingNodes); + + this.reloadTime = TimeUtil.currentTimeMillis(); + + this.lock = new ReentrantReadWriteLock(); + this.blacklistConfig = confInitNew.getBlacklistConfig(); + this.userConfig = userConfig; + this.dbConfig = dbConfig; + this.shardingConfig = shardingConfig; + this.sequenceConfig = sequenceConfig; + } + private void waitIfChanging() { while (changing) { LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10)); @@ -171,9 +212,11 @@ public boolean reload(Map newUsers, Map recycleDbGroups, Map> newErRelations, SystemVariables newSystemVariables, boolean isFullyConfigured, - final int loadAllMode, Map newBlacklistConfig, Map newFunctions) throws SQLNonTransientException { + final int loadAllMode, Map newBlacklistConfig, Map newFunctions, + String userJsonConfig, String sequenceJsonConfig, String shardingJsonConfig, String dbJsonConfig) throws SQLNonTransientException { boolean result = apply(newUsers, newSchemas, newShardingNodes, newDbGroups, recycleDbGroups, newErRelations, - newSystemVariables, isFullyConfigured, loadAllMode, newBlacklistConfig, newFunctions); + newSystemVariables, isFullyConfigured, loadAllMode, newBlacklistConfig, newFunctions, userJsonConfig, + sequenceJsonConfig, shardingJsonConfig, dbJsonConfig); this.reloadTime = TimeUtil.currentTimeMillis(); return result; } @@ -286,7 +329,8 @@ private boolean apply(Map newUsers, Map recycleDbGroups, Map> newErRelations, SystemVariables newSystemVariables, - boolean isFullyConfigured, final int loadAllMode, Map newBlacklistConfig, Map newFunctions) throws SQLNonTransientException { + boolean isFullyConfigured, final int loadAllMode, Map newBlacklistConfig, Map newFunctions, + String userJsonConfig, String sequenceJsonConfig, String shardingJsonConfig, String dbJsonConfig) throws SQLNonTransientException { List> delTables = new ArrayList<>(); List> reloadTables = new ArrayList<>(); List delSchema = new ArrayList<>(); @@ -335,6 +379,10 @@ private boolean apply(Map newUsers, this.erRelations = newErRelations; this.blacklistConfig = newBlacklistConfig; this.functions = newFunctions; + this.userConfig = userJsonConfig; + this.dbConfig = dbJsonConfig; + this.shardingConfig = shardingJsonConfig; + this.sequenceConfig = sequenceJsonConfig; CacheService.getInstance().clearCache(); this.changing = false; if (isFullyConfigured) { @@ -411,10 +459,7 @@ private String changeTablesToString(List> delTables) { return sb.toString(); } - /** - * turned all the config into lowerCase config - */ - public void reviseLowerCase() { + public void reviseLowerCase(String sequenceJson) { //user sharding for (UserConfig uc : users.values()) { @@ -456,7 +501,7 @@ public void reviseLowerCase() { erRelations = newErMap; } - loadSequence(); + loadSequence(sequenceJson); selfChecking0(); } @@ -465,6 +510,14 @@ public void loadSequence() { SequenceManager.load(DbleServer.getInstance().getSystemVariables().isLowerCaseTableNames()); } + public void loadSequence(String sequenceJson) { + if (StringUtil.isEmpty(sequenceJson)) { + loadSequence(); + } else { + SequenceManager.load(DbleServer.getInstance().getSystemVariables().isLowerCaseTableNames(), sequenceJson); + } + } + public void selfChecking0() throws ConfigException { // check 1.user's schemas are all existed in sharding's conf // 2.sharding's conf is not empty @@ -512,6 +565,51 @@ public void selfChecking0() throws ConfigException { } } + public void syncJsonToLocal(boolean isWriteToLocal) throws Exception { + XmlProcessBase xmlProcess = new XmlProcessBase(); + xmlProcess.addParseClass(Shardings.class); + xmlProcess.addParseClass(DbGroups.class); + xmlProcess.addParseClass(Users.class); + // init xml + xmlProcess.initJaxbClass(); + + //sharding + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(Table.class, new TableGsonAdapter()); + Gson gson = gsonBuilder.create(); + ClusterLogic.syncShardingXmlToLocal(this.shardingConfig, xmlProcess, gson, isWriteToLocal); + + //db + ClusterLogic.syncDbXmlToLocal(xmlProcess, this.dbConfig, isWriteToLocal); + + //user + gsonBuilder.registerTypeAdapter(User.class, new UserGsonAdapter()); + gson = gsonBuilder.create(); + ClusterLogic.syncUserXmlToLocal(this.userConfig, xmlProcess, gson, isWriteToLocal); + + //sequence + ClusterLogic.syncSequenceToLocal(this.sequenceConfig, isWriteToLocal); + } + + public String getDbConfig() { + return dbConfig; + } + + public void setDbConfig(String dbConfig) { + this.dbConfig = dbConfig; + } + + public String getShardingConfig() { + return shardingConfig; + } + + public String getUserConfig() { + return userConfig; + } + + public String getSequenceConfig() { + return sequenceConfig; + } } diff --git a/src/main/java/com/actiontech/dble/config/converter/DBConverter.java b/src/main/java/com/actiontech/dble/config/converter/DBConverter.java new file mode 100644 index 0000000000..21798b3714 --- /dev/null +++ b/src/main/java/com/actiontech/dble/config/converter/DBConverter.java @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2016-2021 ActionTech. + * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher. + */ +package com.actiontech.dble.config.converter; + +import com.actiontech.dble.backend.datasource.PhysicalDbGroup; +import com.actiontech.dble.cluster.ClusterLogic; +import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; +import com.actiontech.dble.cluster.zkprocess.entity.Property; +import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBGroup; +import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBInstance; +import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.HeartBeat; +import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; +import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.ProblemReporter; +import com.actiontech.dble.config.Versions; +import com.actiontech.dble.config.loader.xml.XMLDbLoader; +import com.actiontech.dble.config.model.db.DbGroupConfig; +import com.actiontech.dble.config.model.db.DbInstanceConfig; +import com.actiontech.dble.config.model.db.PoolConfig; +import com.actiontech.dble.config.util.ConfigException; +import com.actiontech.dble.config.util.ParameterMapping; +import com.actiontech.dble.util.DecryptUtil; +import com.actiontech.dble.util.StringUtil; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLStreamException; +import java.lang.reflect.InvocationTargetException; +import java.util.*; +import java.util.regex.Matcher; +import java.util.stream.Collectors; + +public class DBConverter { + + private static final Logger LOGGER = LoggerFactory.getLogger(DBConverter.class); + + private final Map dbGroupMap = Maps.newLinkedHashMap(); + + public static String dbXmlToJson() throws JAXBException, XMLStreamException { + XmlProcessBase xmlProcess = new XmlProcessBase(); + xmlProcess.addParseClass(DbGroups.class); + xmlProcess.initJaxbClass(); + String path = ClusterPathUtil.LOCAL_WRITE_PATH + ConfigFileName.DB_XML; + return parseDbGroupXmlFileToJson(xmlProcess, path); + } + + public DbGroups dbJsonToBean(String dbJson) { + return ClusterLogic.parseDbGroupsJsonToBean(new Gson(), dbJson); + } + + public static String dbBeanToJson(DbGroups dbGroups) { + Gson gson = new Gson(); + // bean to json obj + JsonObject jsonObj = new JsonObject(); + jsonObj.addProperty(ClusterPathUtil.VERSION, dbGroups.getVersion()); + + jsonObj.add(ClusterPathUtil.DB_GROUP, gson.toJsonTree(dbGroups.getDbGroup())); + //from json obj to string + return gson.toJson(jsonObj); + } + + public void dbJsonToMap(String dbJson, ProblemReporter problemReporter) { + DbGroups dbs = dbJsonToBean(dbJson); + //check + beanValidate(dbs); + if (dbs.getVersion() != null && !Versions.CONFIG_VERSION.equals(dbs.getVersion())) { + if (problemReporter != null) { + if (Versions.checkVersion(dbs.getVersion())) { + String message = "The dble-config-version is " + Versions.CONFIG_VERSION + ",but the " + + ConfigFileName.DB_XML + " version is " + dbs.getVersion() + ".There may be some incompatible config between two versions, please check it"; + problemReporter.warn(message); + } else { + String message = "The dble-config-version is " + Versions.CONFIG_VERSION + ",but the " + ConfigFileName.DB_XML + " version is " + dbs.getVersion() + ".There must be some incompatible config between two versions, please check it"; + problemReporter.warn(message); + } + } + } + for (DBGroup dbGroup : dbs.getDbGroup()) { + String dbGroupName = dbGroup.getName(); + Matcher nameMatcher = XMLDbLoader.PATTERN_DB.matcher(dbGroupName); + if (!nameMatcher.matches()) { + throw new ConfigException("dbGroup name " + dbGroupName + " show be use " + XMLDbLoader.DB_NAME_FORMAT + "!"); + } + if (this.dbGroupMap.containsKey(dbGroupName)) { + throw new ConfigException("dbGroup name " + dbGroupName + " duplicated!"); + } + List dbInstanceList = dbGroup.getDbInstance(); + int delayThreshold = Optional.ofNullable(dbGroup.getDelayThreshold()).orElse(-1); + String disableHAStr = dbGroup.getDisableHA(); + boolean disableHA = Boolean.parseBoolean(Optional.ofNullable(disableHAStr).orElse("false")); + int rwSplitMode = dbGroup.getRwSplitMode(); + HeartBeat heartbeat = dbGroup.getHeartbeat(); + String heartbeatSQL = heartbeat.getValue(); + + Set instanceNames = new HashSet<>(); + Set instanceUrls = new HashSet<>(); + int readHostSize = dbInstanceList.size() - 1; + DbInstanceConfig writeDbConf = null; + DbInstanceConfig[] readDbConfList = new DbInstanceConfig[readHostSize]; + int readCnt = 0; + for (DBInstance dbInstance : dbInstanceList) { + DbInstanceConfig dbInstanceConfig; + try { + dbInstanceConfig = createDbInstanceConf(dbGroup, dbInstance, problemReporter); + } catch (Exception e) { + throw new ConfigException("db json to map occurred parse errors, The detailed results are as follows . " + e, e); + } + String instanceName = dbInstanceConfig.getInstanceName(); + String instanceUrl = dbInstanceConfig.getUrl(); + Optional.of(instanceName).filter(currentName -> !instanceNames.contains(currentName)).orElseThrow(() -> + new ConfigException("dbGroup[" + dbGroupName + "]'s child host name [" + instanceName + "] duplicated!")); + instanceNames.add(instanceName); + Optional.of(instanceUrl).filter(currentUrl -> !instanceUrls.contains(currentUrl)).orElseThrow(() -> + new ConfigException("dbGroup[" + dbGroupName + "]'s child url [" + instanceUrl + "] duplicated!")); + instanceUrls.add(instanceUrl); + if (dbInstanceConfig.isPrimary()) { + if (writeDbConf == null) { + writeDbConf = dbInstanceConfig; + } else { + throw new ConfigException("dbGroup[" + dbGroupName + "] has multi primary instance!"); + } + } else { + if (readCnt == readHostSize) { + throw new ConfigException("dbGroup[" + dbGroupName + "] has no primary instance!"); + } + readDbConfList[readCnt++] = dbInstanceConfig; + } + } + DbGroupConfig dbGroupConf = new DbGroupConfig(dbGroupName, writeDbConf, readDbConfList, delayThreshold, disableHA); + dbGroupConf.setRwSplitMode(rwSplitMode); + dbGroupConf.setHeartbeatSQL(heartbeatSQL); + int heartbeatTimeout = Optional.ofNullable(heartbeat.getTimeout()).orElse(0); + dbGroupConf.setHeartbeatTimeout(heartbeatTimeout * 1000); + int heartbeatErrorRetryCount = Optional.ofNullable(heartbeat.getErrorRetryCount()).orElse(1); + dbGroupConf.setErrorRetryCount(heartbeatErrorRetryCount); + + PhysicalDbGroup physicalDbGroup = XMLDbLoader.getPhysicalDBPoolSingleWH(dbGroupConf); + this.dbGroupMap.put(dbGroupConf.getName(), physicalDbGroup); + } + } + + private void beanValidate(DbGroups dbs) { + if (null == dbs) { + return; + } + List dbGroupList = dbs.getDbGroup(); + if (dbGroupList == null || dbGroupList.isEmpty()) { + throw new ConfigException("dbGroup is empty"); + } + for (DBGroup dbGroup : dbGroupList) { + if (dbGroup.getDbInstance() == null || dbGroup.getDbInstance().isEmpty()) { + throw new ConfigException("The content of element type \"dbGroup\" is incomplete, it must match \"(heartbeat,dbInstance+)\""); + } + if (dbGroup.getHeartbeat() == null) { + throw new ConfigException("The content of element type \"dbGroup\" is incomplete, it must match \"(heartbeat,dbInstance+)\""); + } + } + } + + static String parseDbGroupXmlFileToJson(XmlProcessBase xmlParseBase, String path) throws JAXBException, XMLStreamException { + // xml file to bean + DbGroups groupsBean; + try { + groupsBean = (DbGroups) xmlParseBase.baseParseXmlToBean(path); + } catch (Exception e) { + LOGGER.warn("parseXmlToBean Exception", e); + throw e; + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Xml to DbGroups is :" + groupsBean); + } + return dbBeanToJson(groupsBean); + } + + private DbInstanceConfig createDbInstanceConf(DBGroup dbGroup, DBInstance dbInstance, ProblemReporter problemReporter) throws InvocationTargetException, IllegalAccessException { + String name = dbInstance.getName(); + String nodeUrl = dbInstance.getUrl(); + String user = dbInstance.getUser(); + String password = dbInstance.getPassword(); + String usingDecryptStr = dbInstance.getUsingDecrypt(); + + Matcher nameMatcher = XMLDbLoader.PATTERN_DB.matcher(name); + if (!nameMatcher.matches()) { + throw new ConfigException("dbInstance name " + name + " show be use " + XMLDbLoader.DB_NAME_FORMAT + "!"); + } + if (StringUtil.isEmpty(name) || StringUtil.isEmpty(nodeUrl) || StringUtil.isEmpty(user)) { + throw new ConfigException( + "dbGroup " + dbGroup.getName() + + " define error,some attributes of this element is empty: " + + name); + } + boolean usingDecrypt = Boolean.parseBoolean(Optional.ofNullable(usingDecryptStr).orElse("false")); + password = DecryptUtil.dbHostDecrypt(usingDecrypt, name, user, password); + String disabledStr = dbInstance.getDisabled(); + boolean disabled = Boolean.parseBoolean(Optional.ofNullable(disabledStr).orElse("false")); + String readWeightStr = dbInstance.getReadWeight(); + List propertyList = dbInstance.getProperty(); + int readWeight = Integer.parseInt(Optional.ofNullable(readWeightStr).orElse("0")); + if (readWeight < 0) { + throw new ConfigException("readWeight attribute in dbInstance[" + name + "] can't be less than 0!"); + } + // init properties of connection pool + PoolConfig poolConfig = new PoolConfig(); + if (!propertyList.isEmpty()) { + Map propertyMap = propertyList.stream().collect(Collectors.toMap(Property::getName, Property::getValue)); + ParameterMapping.mapping(poolConfig, propertyMap, problemReporter); + if (propertyMap.size() > 0) { + throw new ConfigException("These properties of system are not recognized: " + StringUtil.join(propertyMap.keySet(), ",")); + } + } + + Integer maxCon = dbInstance.getMaxCon(); + Integer minCon = dbInstance.getMinCon(); + int colonIndex = nodeUrl.indexOf(':'); + String ip = nodeUrl.substring(0, colonIndex).trim(); + int port = Integer.parseInt(nodeUrl.substring(colonIndex + 1).trim()); + Boolean primary = Optional.ofNullable(dbInstance.getPrimary()).orElse(false); + DbInstanceConfig conf = new DbInstanceConfig(name, ip, port, nodeUrl, user, password, disabled, primary, usingDecrypt); + conf.setMaxCon(maxCon); + conf.setMinCon(minCon); + conf.setReadWeight(readWeight); + // id + String id = dbInstance.getId(); + if (StringUtil.isEmpty(id)) { + conf.setId(name); + } else { + conf.setId(id); + } + conf.setPoolConfig(poolConfig); + return conf; + } + + public Map getDbGroupMap() { + return dbGroupMap; + } +} diff --git a/src/main/java/com/actiontech/dble/config/converter/SequenceConverter.java b/src/main/java/com/actiontech/dble/config/converter/SequenceConverter.java new file mode 100644 index 0000000000..6e9ab8355e --- /dev/null +++ b/src/main/java/com/actiontech/dble/config/converter/SequenceConverter.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2016-2021 ActionTech. + * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher. + */ +package com.actiontech.dble.config.converter; + +import com.actiontech.dble.route.util.PropertiesUtil; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.util.Map; +import java.util.Properties; + +public class SequenceConverter { + + private String fileName; + + public static String sequencePropsToJson(String fileName) { + JsonObject jsonObject = new JsonObject(); + Properties properties = PropertiesUtil.loadProps(fileName); + jsonObject.addProperty(fileName, (new Gson()).toJson(properties)); + return (new Gson()).toJson(jsonObject); + } + + public Properties jsonToProperties(String sequenceJson) { + JsonObject jsonObj = new JsonParser().parse(sequenceJson).getAsJsonObject(); + Map.Entry sequenceEntry = jsonObj.entrySet().iterator().next(); + if (null == sequenceEntry) { + return null; + } + this.fileName = sequenceEntry.getKey(); + JsonElement fileContent = sequenceEntry.getValue(); + JsonObject fileContentJson = new JsonParser().parse(fileContent.getAsString()).getAsJsonObject(); + Properties props = new Properties(); + fileContentJson.entrySet().forEach(jsonEntry -> props.setProperty(jsonEntry.getKey(), jsonEntry.getValue().getAsString())); + return props; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } +} diff --git a/src/main/java/com/actiontech/dble/config/converter/ShardingConverter.java b/src/main/java/com/actiontech/dble/config/converter/ShardingConverter.java new file mode 100644 index 0000000000..1f589b16bf --- /dev/null +++ b/src/main/java/com/actiontech/dble/config/converter/ShardingConverter.java @@ -0,0 +1,700 @@ +/* + * Copyright (C) 2016-2021 ActionTech. + * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher. + */ +package com.actiontech.dble.config.converter; + +import com.actiontech.dble.backend.datasource.PhysicalDbGroup; +import com.actiontech.dble.cluster.ClusterLogic; +import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.zkprocess.comm.ConfFileRWUtils; +import com.actiontech.dble.cluster.zkprocess.console.ParseParamEnum; +import com.actiontech.dble.cluster.zkprocess.entity.Property; +import com.actiontech.dble.cluster.zkprocess.entity.Shardings; +import com.actiontech.dble.cluster.zkprocess.entity.sharding.function.Function; +import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.*; +import com.actiontech.dble.cluster.zkprocess.entity.sharding.shardingnode.ShardingNode; +import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; +import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.ErrorInfo; +import com.actiontech.dble.config.ProblemReporter; +import com.actiontech.dble.config.Versions; +import com.actiontech.dble.config.loader.xml.XMLShardingLoader; +import com.actiontech.dble.config.model.ClusterConfig; +import com.actiontech.dble.config.model.sharding.SchemaConfig; +import com.actiontech.dble.config.model.sharding.ShardingNodeConfig; +import com.actiontech.dble.config.model.sharding.table.*; +import com.actiontech.dble.config.util.ConfigException; +import com.actiontech.dble.config.util.ParameterMapping; +import com.actiontech.dble.route.function.*; +import com.actiontech.dble.route.sequence.handler.IncrSequenceMySQLHandler; +import com.actiontech.dble.util.SplitUtil; +import com.actiontech.dble.util.StringUtil; +import com.google.common.collect.Maps; +import com.google.gson.*; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLStreamException; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.actiontech.dble.backend.datasource.check.GlobalCheckJob.GLOBAL_TABLE_CHECK_DEFAULT; +import static com.actiontech.dble.backend.datasource.check.GlobalCheckJob.GLOBAL_TABLE_CHECK_DEFAULT_CRON; + +public class ShardingConverter { + private static final Logger LOGGER = LoggerFactory.getLogger(ShardingConverter.class); + + private Map shardingNodeMap = Maps.newLinkedHashMap(); + private final Map functionMap = Maps.newLinkedHashMap(); + private final Map schemaConfigMap = Maps.newLinkedHashMap(); + private final Map> erRelations = Maps.newLinkedHashMap(); + private final AtomicInteger tableIndex = new AtomicInteger(0); + private final Gson gson; + + public ShardingConverter() { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(Table.class, new TableGsonAdapter()); + this.gson = gsonBuilder.create(); + } + + public String shardingXmlToJson() throws JAXBException, XMLStreamException { + XmlProcessBase xmlProcess = new XmlProcessBase(); + xmlProcess.addParseClass(Shardings.class); + xmlProcess.initJaxbClass(); + String path = ClusterPathUtil.LOCAL_WRITE_PATH + ConfigFileName.SHARDING_XML; + String json = parseShardingXmlFileToJson(xmlProcess, path); + return json; + } + + public Shardings shardingJsonToBean(String shardingJson) { + return ClusterLogic.parseShardingJsonToBean(gson, shardingJson); + } + + public String shardingBeanToJson(Shardings shardings) { + // bean to json obj + JsonObject jsonObj = new JsonObject(); + jsonObj.addProperty(ClusterPathUtil.VERSION, shardings.getVersion()); + + JsonArray schemaArray = new JsonArray(); + for (Schema schema : shardings.getSchema()) { + if (schema.getTable() != null) { + JsonObject schemaJsonObj = gson.toJsonTree(schema).getAsJsonObject(); + schemaJsonObj.remove("table"); + JsonArray tableArray = new JsonArray(); + for (Object table : schema.getTable()) { + JsonElement tableElement = gson.toJsonTree(table, Table.class); + tableArray.add(tableElement); + } + schemaJsonObj.add("table", gson.toJsonTree(tableArray)); + schemaArray.add(gson.toJsonTree(schemaJsonObj)); + } else { + schemaArray.add(gson.toJsonTree(schema)); + } + } + jsonObj.add(ClusterPathUtil.SCHEMA, gson.toJsonTree(schemaArray)); + jsonObj.add(ClusterPathUtil.SHARDING_NODE, gson.toJsonTree(shardings.getShardingNode())); + List functionList = shardings.getFunction(); + readMapFileAddFunction(functionList); + jsonObj.add(ClusterPathUtil.FUNCTION, gson.toJsonTree(functionList)); + //from json obj to string + return gson.toJson(jsonObj); + } + + public void shardingJsonToMap(String shardingJson, Map dbGroupMap, String sequenceJson, ProblemReporter problemReporter) { + Shardings shardings = shardingJsonToBean(shardingJson); + List shardingNodeList = shardings.getShardingNode(); + List functionList = shardings.getFunction(); + removeFileContent(functionList); + List schemaList = shardings.getSchema(); + Map shardingNodeConfigMap = Maps.newLinkedHashMap(); + List errorInfos = new ArrayList<>(); + if (shardings.getVersion() != null && !Versions.CONFIG_VERSION.equals(shardings.getVersion())) { + if (problemReporter != null) { + if (Versions.checkVersion(shardings.getVersion())) { + String message = "The dble-config-version is " + Versions.CONFIG_VERSION + ",but the " + ConfigFileName.SHARDING_XML + " version is " + shardings.getVersion() + ".There may be some incompatible config between two versions, please check it"; + problemReporter.warn(message); + } else { + String message = "The dble-config-version is " + Versions.CONFIG_VERSION + ",but the " + ConfigFileName.SHARDING_XML + " version is " + shardings.getVersion() + ".There must be some incompatible config between two versions, please check it"; + problemReporter.warn(message); + } + } + } + try { + shardingNodeListToMap(shardingNodeList, dbGroupMap, shardingNodeConfigMap); + functionListToMap(functionList, problemReporter); + schemaListToMap(schemaList, shardingNodeConfigMap, problemReporter); + deleteUselessShardingNode(errorInfos, sequenceJson); + } catch (Exception e) { + throw new ConfigException("sharding json to map occurred parse errors, The detailed errors are as follows . " + e, e); + } + } + + String parseShardingXmlFileToJson(XmlProcessBase xmlParseBase, String path) throws JAXBException, XMLStreamException { + // xml file to bean + Shardings shardingBean; + try { + shardingBean = (Shardings) xmlParseBase.baseParseXmlToBean(path); + } catch (Exception e) { + LOGGER.warn("parseXmlToBean Exception", e); + throw e; + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Xml to Shardings is :" + shardingBean); + } + return shardingBeanToJson(shardingBean); + } + + private static void readMapFileAddFunction(List functionList) { + List tempData = new ArrayList<>(); + for (Function function : functionList) { + List proList = function.getProperty(); + if (null != proList && !proList.isEmpty()) { + for (Property property : proList) { + // if mapfile,read and save to json + if (ParseParamEnum.ZK_PATH_RULE_MAPFILE_NAME.getKey().equals(property.getName())) { + Property mapFilePro = new Property(); + mapFilePro.setName(property.getValue()); + try { + mapFilePro.setValue(ConfFileRWUtils.readFile(property.getValue())); + tempData.add(mapFilePro); + } catch (IOException e) { + LOGGER.warn("readMapFile IOException", e); + } + } + } + proList.addAll(tempData); + tempData.clear(); + } + } + } + + private void schemaListToMap(List schemaList, Map shardingNodeConfigMap, ProblemReporter problemReporter) { + Map> funcNodeERMap = Maps.newLinkedHashMap(); + for (com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.Schema schema : schemaList) { + String schemaName = schema.getName(); + String schemaShardingNode = schema.getShardingNode(); + String schemaSqlMaxLimitStr = null == schema.getSqlMaxLimit() ? null : String.valueOf(schema.getSqlMaxLimit()); + List tableList = Optional.ofNullable(schema.getTable()).orElse(Collections.EMPTY_LIST); + + int schemaSqlMaxLimit = XMLShardingLoader.getSqlMaxLimit(schemaSqlMaxLimitStr, -1); + //check and add shardingNode + if (schemaShardingNode != null && !schemaShardingNode.isEmpty()) { + List shardingNodeLst = new ArrayList<>(1); + shardingNodeLst.add(schemaShardingNode); + checkShardingNodeExists(shardingNodeLst, shardingNodeConfigMap); + } else { + schemaShardingNode = null; + } + //load tables from sharding + Map tableConfigMap = Maps.newLinkedHashMap(); + if (this.schemaConfigMap.containsKey(schemaName)) { + throw new ConfigException("schema " + schemaName + " duplicated!"); + } + + for (Object tableObj : tableList) { + if (tableObj instanceof ShardingTable) { + fillShardingTable((ShardingTable) tableObj, schemaSqlMaxLimit, tableConfigMap, shardingNodeConfigMap, problemReporter); + } else if (tableObj instanceof GlobalTable) { + fillGlobalTable((GlobalTable) tableObj, schemaSqlMaxLimit, tableConfigMap, shardingNodeConfigMap); + } else if (tableObj instanceof SingleTable) { + fillSingleTable((SingleTable) tableObj, schemaSqlMaxLimit, tableConfigMap, shardingNodeConfigMap); + } + } + + // if sharding has no default shardingNode,it must contains at least one table + if (schemaShardingNode == null && tableConfigMap.size() == 0) { + throw new ConfigException( + "sharding " + schemaName + " didn't config tables,so you must set shardingNode property!"); + } + SchemaConfig schemaConfig = new SchemaConfig(schemaName, schemaShardingNode, tableConfigMap, schemaSqlMaxLimit); + mergeFuncNodeERMap(schemaConfig, funcNodeERMap); + mergeFkERMap(schemaConfig); + this.schemaConfigMap.put(schemaName, schemaConfig); + } + makeAllErRelations(funcNodeERMap); + } + + public static void removeFileContent(List functionList) { + if (functionList == null) { + return; + } + List tempData = new ArrayList<>(); + List writeData = new ArrayList<>(); + for (Function function : functionList) { + List proList = function.getProperty(); + if (null != proList && !proList.isEmpty()) { + for (Property property : proList) { + if (ParseParamEnum.ZK_PATH_RULE_MAPFILE_NAME.getKey().equals(property.getName())) { + tempData.add(property); + } + } + + if (!tempData.isEmpty()) { + for (Property property : tempData) { + for (Property prozkdownload : proList) { + if (property.getValue().equals(prozkdownload.getName())) { + writeData.add(prozkdownload); + } + } + } + } + + proList.removeAll(writeData); + + tempData.clear(); + writeData.clear(); + } + } + + } + + private void fillSingleTable(SingleTable singleTable, int schemaSqlMaxLimit, Map tableConfigMap, Map shardingNodeConfigMap) { + String singleTableName = singleTable.getName(); + String singleTableSqlMaxLimitStr = null == singleTable.getSqlMaxLimit() ? null : String.valueOf(singleTable.getSqlMaxLimit()); + String singleTableShardingNode = singleTable.getShardingNode(); + + if (StringUtil.isBlank(singleTableName)) { + throw new ConfigException("one of tables' name is empty"); + } + //limit size of the table + int tableSqlMaxLimit = XMLShardingLoader.getSqlMaxLimit(singleTableSqlMaxLimitStr, schemaSqlMaxLimit); + if (StringUtil.isBlank(singleTableShardingNode)) { + throw new ConfigException("shardingNode of " + singleTableName + " is empty"); + } + String[] theShardingNodes = SplitUtil.split(singleTableShardingNode, ',', '$', '-'); + if (theShardingNodes.length != 1) { + throw new ConfigException("invalid shardingNode config: " + singleTableShardingNode + " for SingleTableConfig " + singleTableName); + } + String[] tableNames = singleTableName.split(","); + + for (String tableName : tableNames) { + if (tableName.contains("`")) { + tableName = tableName.replaceAll("`", ""); + } + if (StringUtil.isBlank(tableName)) { + throw new ConfigException("one of table name of " + singleTableName + " is empty"); + } + SingleTableConfig table = new SingleTableConfig(tableName, tableSqlMaxLimit, Arrays.asList(theShardingNodes)); + checkShardingNodeExists(table.getShardingNodes(), shardingNodeConfigMap); + if (tableConfigMap.containsKey(table.getName())) { + throw new ConfigException("table " + tableName + " duplicated!"); + } + table.setId(this.tableIndex.incrementAndGet()); + tableConfigMap.put(table.getName(), table); + } + } + + private void fillGlobalTable(GlobalTable globalTable, int schemaSqlMaxLimit, Map tableConfigMap, Map shardingNodeConfigMap) { + String globalTableName = globalTable.getName(); + String globalTableSqlMaxLimitStr = null == globalTable.getSqlMaxLimit() ? null : String.valueOf(globalTable.getSqlMaxLimit()); + String globalTableShardingNode = globalTable.getShardingNode(); + String globalTableCheckClass = Optional.ofNullable(globalTable.getCheckClass()).orElse(GLOBAL_TABLE_CHECK_DEFAULT); + String globalTableCron = Optional.ofNullable(globalTable.getCron()).orElse(GLOBAL_TABLE_CHECK_DEFAULT_CRON).toUpperCase(); + boolean globalCheck = !StringUtil.isBlank(globalTable.getCheckClass()); + + if (StringUtil.isBlank(globalTableName)) { + throw new ConfigException("one of tables' name is empty"); + } + //limit size of the table + int tableSqlMaxLimit = XMLShardingLoader.getSqlMaxLimit(globalTableSqlMaxLimitStr, schemaSqlMaxLimit); + if (StringUtil.isBlank(globalTableShardingNode)) { + throw new ConfigException("shardingNode of " + globalTableName + " is empty"); + } + String[] theShardingNodes = SplitUtil.split(globalTableShardingNode, ',', '$', '-'); + final long distinctCount = Arrays.stream(theShardingNodes).distinct().count(); + if (distinctCount != theShardingNodes.length) { + //detected repeat props; + throw new ConfigException("invalid shardingNode config: " + globalTableShardingNode + " for GlobalTableConfig " + globalTableName + ",the nodes duplicated!"); + } + if (theShardingNodes.length <= 1) { + throw new ConfigException("invalid shardingNode config: " + globalTableShardingNode + " for GlobalTableConfig " + globalTableName + ", please use SingleTable"); + } + String[] tableNames = globalTableName.split(","); + + for (String tableName : tableNames) { + if (tableName.contains("`")) { + tableName = tableName.replaceAll("`", ""); + } + if (StringUtil.isBlank(tableName)) { + throw new ConfigException("one of table name of " + globalTableName + " is empty"); + } + GlobalTableConfig table = new GlobalTableConfig(tableName, tableSqlMaxLimit, Arrays.asList(theShardingNodes), + globalTableCron, globalTableCheckClass, globalCheck); + checkShardingNodeExists(table.getShardingNodes(), shardingNodeConfigMap); + if (tableConfigMap.containsKey(table.getName())) { + throw new ConfigException("table " + tableName + " duplicated!"); + } + table.setId(this.tableIndex.incrementAndGet()); + tableConfigMap.put(table.getName(), table); + } + } + + private void fillShardingTable(ShardingTable shardingTable, int schemaSqlMaxLimit, Map tableConfigMap, Map shardingNodeConfigMap, ProblemReporter problemReporter) { + String shardingTableName = shardingTable.getName(); + String shardingTableSqlMaxLimitStr = null == shardingTable.getSqlMaxLimit() ? null : String.valueOf(shardingTable.getSqlMaxLimit()); + String shardingTableShardingColumn = shardingTable.getShardingColumn(); + boolean shardingTableSqlRequiredSharding = Optional.ofNullable(shardingTable.getSqlRequiredSharding()).orElse(false); + + if (StringUtil.isBlank(shardingTableName)) { + throw new ConfigException("one of tables' name is empty"); + } + int tableSqlMaxLimit = XMLShardingLoader.getSqlMaxLimit(shardingTableSqlMaxLimitStr, schemaSqlMaxLimit); + //shardingNode of table + if (StringUtil.isBlank(shardingTableShardingColumn)) { + throw new ConfigException("shardingColumn of " + shardingTableName + " is empty"); + } + shardingTableShardingColumn = shardingTableShardingColumn.toUpperCase(); + String shardingTableIncrementColumn = StringUtil.isBlank(shardingTable.getIncrementColumn()) ? null : shardingTable.getIncrementColumn().toUpperCase(); + String shardingTableFunction = shardingTable.getFunction(); + if (StringUtil.isBlank(shardingTableFunction)) { + throw new ConfigException("function of " + shardingTableName + " is empty"); + } + AbstractPartitionAlgorithm algorithm = this.functionMap.get(shardingTableFunction); + if (algorithm == null) { + throw new ConfigException("can't find function of name :" + shardingTableFunction + " in table " + shardingTableName); + } + String shardingTableShardingNode = shardingTable.getShardingNode(); + if (StringUtil.isBlank(shardingTableShardingNode)) { + throw new ConfigException("shardingNode of " + shardingTableName + " is empty"); + } + String[] theShardingNodes = SplitUtil.split(shardingTableShardingNode, ',', '$', '-'); + final long distinctCount = Arrays.stream(theShardingNodes).distinct().count(); + if (distinctCount != theShardingNodes.length) { + //detected repeat props; + + throw new ConfigException("invalid shardingNode config: " + shardingTableShardingNode + " for ShardingTableConfig " + shardingTableName + " ,the nodes duplicated!"); + } + if (theShardingNodes.length <= 1) { + throw new ConfigException("invalid shardingNode config: " + shardingTableShardingNode + " for ShardingTableConfig " + shardingTableName + ", please use SingleTable"); + } + List lstShardingNode = Arrays.asList(theShardingNodes); + String[] tableNames = shardingTableName.split(","); + + for (String tableName : tableNames) { + if (tableName.contains("`")) { + tableName = tableName.replaceAll("`", ""); + } + if (StringUtil.isBlank(tableName)) { + throw new ConfigException("one of table name of " + shardingTableName + " is empty"); + } + ShardingTableConfig table = new ShardingTableConfig(tableName, tableSqlMaxLimit, + lstShardingNode, shardingTableIncrementColumn, algorithm, shardingTableShardingColumn, shardingTableSqlRequiredSharding); + checkShardingNodeExists(table.getShardingNodes(), shardingNodeConfigMap); + checkRuleSuitTable(table, shardingTableFunction, problemReporter); + if (tableConfigMap.containsKey(table.getName())) { + throw new ConfigException("table " + tableName + " duplicated!"); + } + table.setId(this.tableIndex.incrementAndGet()); + tableConfigMap.put(table.getName(), table); + } + // child table must know its unique father + if (tableNames.length == 1) { + ShardingTableConfig parentTable = (ShardingTableConfig) (tableConfigMap.get(tableNames[0])); + // process child tables + final List childTableList = shardingTable.getChildTable(); + processChildTables(tableConfigMap, parentTable, lstShardingNode, childTableList, schemaSqlMaxLimit); + } + } + + private void makeAllErRelations(Map> funcNodeERMap) { + if (funcNodeERMap == null) { + return; + } + Iterator>> iterator = funcNodeERMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + if (entry.getValue().size() == 1) { + iterator.remove(); + continue; + } + for (ERTable erTable : entry.getValue()) { + Set relations = this.erRelations.get(erTable); + if (relations == null) { + this.erRelations.put(erTable, entry.getValue()); + } else { + relations.addAll(entry.getValue()); + } + } + } + } + + + private void deleteUselessShardingNode(List errorInfos, String sequenceJson) { + Set allUseShardingNode = new HashSet<>(); + for (SchemaConfig sc : this.schemaConfigMap.values()) { + // check shardingNode / dbGroup + Set shardingNodeNames = sc.getAllShardingNodes(); + allUseShardingNode.addAll(shardingNodeNames); + } + + // add global sequence node when it is some dedicated servers */ + if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_MYSQL && !StringUtil.isBlank(sequenceJson)) { + IncrSequenceMySQLHandler redundancy = new IncrSequenceMySQLHandler(); + redundancy.loadByJson(false, sequenceJson); + allUseShardingNode.addAll(redundancy.getShardingNodes()); + } + + //delete redundancy shardingNode + Iterator> iterator = this.shardingNodeMap.entrySet().iterator(); + PhysicalDbGroup shardingNodeGroup; + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String shardingNodeName = entry.getKey(); + if (allUseShardingNode.contains(shardingNodeName)) { + shardingNodeGroup = entry.getValue().getDbGroup(); + if (shardingNodeGroup != null) { + shardingNodeGroup.setShardingUseless(false); + } else { + throw new ConfigException("The dbGroup[" + entry.getValue().getDbGroupName() + "] associated with ShardingNode[" + entry.getKey() + "] does not exist"); + } + } else { + errorInfos.add(new ErrorInfo("Xml", "WARNING", "shardingNode " + shardingNodeName + " is useless")); + iterator.remove(); + } + } + } + + private void mergeFkERMap(SchemaConfig schemaConfig) { + Map> schemaFkERMap = schemaConfig.getFkErRelations(); + if (schemaFkERMap == null) { + return; + } + this.erRelations.putAll(schemaFkERMap); + } + + private void mergeFuncNodeERMap(SchemaConfig schemaConfig, Map> funcNodeERMap) { + Map> schemaFuncNodeER = schemaConfig.getFuncNodeERMap(); + if (schemaFuncNodeER == null) { + return; + } + for (Map.Entry> entry : schemaFuncNodeER.entrySet()) { + String key = entry.getKey(); + if (funcNodeERMap == null) { + funcNodeERMap = new HashMap<>(); + } + if (!funcNodeERMap.containsKey(key)) { + funcNodeERMap.put(key, entry.getValue()); + } else { + Set setFuncNode = funcNodeERMap.get(key); + setFuncNode.addAll(entry.getValue()); + } + } + } + + private void processChildTables(Map tables, BaseTableConfig parentTable, List lstShardingNode, List shardingTable, + int schemaSqlMaxLimit) { + for (ChildTable childTable : shardingTable) { + String childTableName = childTable.getName(); + String childTableSqlMaxLimitStr = null == childTable.getSqlMaxLimit() ? null : String.valueOf(childTable.getSqlMaxLimit()); + String childTableJoinColumn = childTable.getJoinColumn().toUpperCase(); + String childTableParentColumn = childTable.getParentColumn().toUpperCase(); + String childTableIncrementColumn = StringUtil.isBlank(childTable.getIncrementColumn()) ? null : childTable.getIncrementColumn().toUpperCase(); + + if (StringUtil.isBlank(childTableName)) { + throw new ConfigException("one of table [" + parentTable.getName() + "]'s child name is empty"); + } + int tableSqlMaxLimit = XMLShardingLoader.getSqlMaxLimit(childTableSqlMaxLimitStr, schemaSqlMaxLimit); + + ChildTableConfig table = new ChildTableConfig(childTableName, tableSqlMaxLimit, lstShardingNode, + parentTable, childTableJoinColumn, childTableParentColumn, childTableIncrementColumn); + + if (tables.containsKey(table.getName())) { + throw new ConfigException("table " + table.getName() + " duplicated!"); + } + table.setId(this.tableIndex.incrementAndGet()); + tables.put(table.getName(), table); + //child table may also have children + List childTableList = childTable.getChildTable(); + processChildTables(tables, table, lstShardingNode, childTableList, schemaSqlMaxLimit); + } + + } + + private void checkShardingNodeExists(List nodes, Map shardingNodeConfigMap) { + if (nodes == null || nodes.size() < 1) { + return; + } + for (String node : nodes) { + if (!shardingNodeConfigMap.containsKey(node)) { + throw new ConfigException("shardingNode '" + node + "' is not found!"); + } + } + } + + private void checkRuleSuitTable(ShardingTableConfig tableConf, String functionName, ProblemReporter problemReporter) { + AbstractPartitionAlgorithm function = tableConf.getFunction(); + int suitValue = function.suitableFor(tableConf.getShardingNodes().size()); + if (suitValue < 0) { + throw new ConfigException("Illegal table conf : table [ " + tableConf.getName() + " ] rule function [ " + + functionName + " ] partition size : " + tableConf.getShardingColumn() + " > table shardingNode size : " + + tableConf.getShardingNodes().size() + ", please make sure table shardingnode size = function partition size"); + } else if (suitValue > 0) { + problemReporter.warn("table conf : table [ " + tableConf.getName() + " ] rule function [ " + functionName + " ] " + + "partition size : " + tableConf.getFunction().getPartitionNum() + " < table shardingNode size : " + tableConf.getShardingNodes().size()); + } + } + + private void functionListToMap(List functionList, ProblemReporter problemReporter) throws InstantiationException, IllegalAccessException, ClassNotFoundException, InvocationTargetException { + for (Function function : functionList) { + String functionName = function.getName(); + String functionClazz = function.getClazz(); + List propertyList = function.getProperty(); + + // check if the function is duplicate + if (this.functionMap.containsKey(functionName)) { + throw new ConfigException("rule function " + functionName + " duplicated!"); + } + //reflection + AbstractPartitionAlgorithm functionInstance = createFunction(functionName, functionClazz); + functionInstance.setName(functionName); + Properties props = new Properties(); + propertyList.forEach(property -> props.put(property.getName(), property.getValue())); + ParameterMapping.mapping(functionInstance, props, problemReporter); + if (props.size() > 0) { + String[] propItem = new String[props.size()]; + props.keySet().toArray(propItem); + throw new ConfigException("These properties of functionInstance [" + functionName + "] is not recognized: " + StringUtil.join(propItem, ",")); + } + + //init for AbstractPartitionAlgorithm + functionInstance.selfCheck(); + functionInstance.init(); + this.functionMap.put(functionName, functionInstance); + } + setFunctionAlias(); + } + + private void setFunctionAlias() { + Map funcAlias = new HashMap<>(); + int i = 0; + for (AbstractPartitionAlgorithm function : this.functionMap.values()) { + String alias = funcAlias.get(function); + if (alias != null) { + function.setAlias(alias); + } else { + alias = "function" + i; + i++; + function.setAlias(alias); + funcAlias.put(function, alias); + } + } + } + + private AbstractPartitionAlgorithm createFunction(String name, String clazz) + throws ClassNotFoundException, InstantiationException, + IllegalAccessException { + + String lowerClass = clazz.toLowerCase(); + switch (lowerClass) { + case "hash": + return new PartitionByLong(); + case "stringhash": + return new PartitionByString(); + case "enum": + return new PartitionByFileMap(); + case "jumpstringhash": + return new PartitionByJumpConsistentHash(); + case "numberrange": + return new AutoPartitionByLong(); + case "patternrange": + return new PartitionByPattern(); + case "date": + return new PartitionByDate(); + default: + Class clz = Class.forName(clazz); + //all function must be extend from AbstractPartitionAlgorithm + if (!AbstractPartitionAlgorithm.class.isAssignableFrom(clz)) { + throw new ConfigException("rule function must implements " + + AbstractPartitionAlgorithm.class.getName() + ", name=" + name); + } + return (AbstractPartitionAlgorithm) clz.newInstance(); + } + + } + + private void shardingNodeListToMap(List shardingNodeList, Map dbGroupMap, Map shardingNodeConfigMap) { + Set checkSet = new HashSet<>(); + for (ShardingNode shardingNode : shardingNodeList) { + String shardingNodeName = shardingNode.getName(); + String shardingNodeDatabase = shardingNode.getDatabase(); + String shardingNodeDbGroup = shardingNode.getDbGroup(); + + if (StringUtils.isBlank(shardingNodeName) || StringUtils.isBlank(shardingNodeDatabase) || StringUtils.isBlank(shardingNodeDbGroup)) { + throw new ConfigException("shardingNode " + shardingNodeName + " define error ,attribute can't be empty"); + } + //dnNamePre(name),databaseStr(database),host(dbGroup) can use ',', '$', '-' to configure multi nodes + // but the database size *dbGroup size must equal the size of name + // every dbGroup has all database in its tag + //eg: + //means:localhost1 has database of db$0-75,localhost2 has database of db$0-75(name is dn$76-151) + String[] dnNames = SplitUtil.split(shardingNodeName, ',', '$', '-'); + String[] databases = SplitUtil.split(shardingNodeDatabase, ',', '$', '-'); + String[] hostStrings = SplitUtil.split(shardingNodeDbGroup, ',', '$', '-'); + + if (dnNames.length != databases.length * hostStrings.length) { + throw new ConfigException("shardingNode " + shardingNodeName + + " define error ,Number of shardingNode name must be = Number of database * Number of dbGroup"); + } + if (dnNames.length > 1) { + List mhdList = XMLShardingLoader.mergerHostDatabase(hostStrings, databases); + for (int k = 0; k < dnNames.length; k++) { + String[] hd = mhdList.get(k); + String dnName = dnNames[k]; + String databaseName = hd[1]; + String hostName = hd[0]; + createSharingNode(dnName, databaseName, hostName, checkSet, shardingNodeConfigMap); + } + } else { + createSharingNode(shardingNodeName, shardingNodeDatabase, shardingNodeDbGroup, checkSet, shardingNodeConfigMap); + } + } + this.shardingNodeMap = initShardingNodes(shardingNodeConfigMap, dbGroupMap); + } + + private void createSharingNode(String dnName, String database, String host, Set checkSet, Map shardingNodeConfigMap) { + + ShardingNodeConfig conf = new ShardingNodeConfig(dnName, database, host); + if (checkSet.contains(host + "#" + database)) { + throw new ConfigException("shardingNode " + conf.getName() + " use the same dbGroup&database with other shardingNode"); + } else { + checkSet.add(host + "#" + database); + } + if (shardingNodeConfigMap.containsKey(conf.getName())) { + throw new ConfigException("shardingNode " + conf.getName() + " duplicated!"); + } + shardingNodeConfigMap.put(conf.getName(), conf); + } + + private Map initShardingNodes(Map nodeConf, Map dbGroupMap) { + Map nodes = new HashMap<>(nodeConf.size()); + for (ShardingNodeConfig conf : nodeConf.values()) { + PhysicalDbGroup pool = dbGroupMap.get(conf.getDbGroupName()); + com.actiontech.dble.backend.datasource.ShardingNode shardingNode = new com.actiontech.dble.backend.datasource.ShardingNode(conf.getDbGroupName(), conf.getName(), conf.getDatabase(), pool); + nodes.put(shardingNode.getName(), shardingNode); + } + return nodes; + } + + + public Map getShardingNodeMap() { + return shardingNodeMap; + } + + public Map getFunctionMap() { + return functionMap; + } + + public Map getSchemaConfigMap() { + return schemaConfigMap; + } + + public Map> getErRelations() { + return erRelations; + } +} diff --git a/src/main/java/com/actiontech/dble/config/converter/UserConverter.java b/src/main/java/com/actiontech/dble/config/converter/UserConverter.java new file mode 100644 index 0000000000..c2d86373db --- /dev/null +++ b/src/main/java/com/actiontech/dble/config/converter/UserConverter.java @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2016-2021 ActionTech. + * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher. + */ +package com.actiontech.dble.config.converter; + +import com.actiontech.dble.cluster.ClusterLogic; +import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.zkprocess.entity.Property; +import com.actiontech.dble.cluster.zkprocess.entity.Users; +import com.actiontech.dble.cluster.zkprocess.entity.user.*; +import com.actiontech.dble.cluster.zkprocess.entity.user.privilege.Schema; +import com.actiontech.dble.cluster.zkprocess.entity.user.privilege.Table; +import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; +import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.ProblemReporter; +import com.actiontech.dble.config.Versions; +import com.actiontech.dble.config.loader.xml.XMLUserLoader; +import com.actiontech.dble.config.model.user.*; +import com.actiontech.dble.config.util.ConfigException; +import com.actiontech.dble.config.util.ParameterMapping; +import com.actiontech.dble.util.DecryptUtil; +import com.actiontech.dble.util.IPAddressUtil; +import com.actiontech.dble.util.SplitUtil; +import com.actiontech.dble.util.StringUtil; +import com.alibaba.druid.wall.WallConfig; +import com.alibaba.druid.wall.WallProvider; +import com.alibaba.druid.wall.spi.MySqlWallProvider; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLStreamException; +import java.lang.reflect.InvocationTargetException; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class UserConverter { + + private static final Logger LOGGER = LoggerFactory.getLogger(UserConverter.class); + private final Gson gson; + private final Map userConfigMap = Maps.newLinkedHashMap(); + private final Map blackListConfigMap = Maps.newLinkedHashMap(); + private final AtomicInteger userId = new AtomicInteger(0); + + public UserConverter() { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(User.class, new UserGsonAdapter()); + this.gson = gsonBuilder.create(); + } + + public Users userJsonToBean(String userJson) { + return ClusterLogic.parseUserJsonToBean(this.gson, userJson); + } + + public String userBeanToJson(Users users) { + // bean to json obj + JsonObject jsonObj = new JsonObject(); + jsonObj.addProperty(ClusterPathUtil.VERSION, users.getVersion()); + + JsonArray userArray = new JsonArray(); + for (Object user : users.getUser()) { + JsonElement tableElement = this.gson.toJsonTree(user, User.class); + userArray.add(tableElement); + } + jsonObj.add(ClusterPathUtil.USER, this.gson.toJsonTree(userArray)); + jsonObj.add(ClusterPathUtil.BLACKLIST, this.gson.toJsonTree(users.getBlacklist())); + //from json obj to string + return this.gson.toJson(jsonObj); + } + + public String userXmlToJson() throws JAXBException, XMLStreamException { + XmlProcessBase xmlProcess = new XmlProcessBase(); + xmlProcess.addParseClass(Users.class); + xmlProcess.initJaxbClass(); + String path = ClusterPathUtil.LOCAL_WRITE_PATH + ConfigFileName.USER_XML; + String json = parseUserXmlFileToJson(xmlProcess, path); + return json; + } + + public void userJsonToMap(String userJson, ProblemReporter problemReporter) { + Users users = userJsonToBean(userJson); + List blacklist = Optional.ofNullable(users.getBlacklist()).orElse(Lists.newArrayList()); + if (users.getVersion() != null && !Versions.CONFIG_VERSION.equals(users.getVersion())) { + if (problemReporter != null) { + if (Versions.checkVersion(users.getVersion())) { + String message = "The dble-config-version is " + Versions.CONFIG_VERSION + ",but the " + ConfigFileName.USER_XML + " version is " + users.getVersion() + ".There may be some incompatible config between two versions, please check it"; + problemReporter.warn(message); + } else { + String message = "The dble-config-version is " + Versions.CONFIG_VERSION + ",but the " + ConfigFileName.USER_XML + " version is " + users.getVersion() + ".There must be some incompatible config between two versions, please check it"; + problemReporter.warn(message); + } + } + } + Map blackListMap; + try { + blackListMap = blackListToMap(blacklist, problemReporter); + userListToMap(users.getUser(), blackListMap, problemReporter); + } catch (Exception e) { + throw new ConfigException("user json to map occurred parse errors, The detailed results are as follows . " + e, e); + } + } + + private String parseUserXmlFileToJson(XmlProcessBase xmlParseBase, String path) throws JAXBException, XMLStreamException { + // xml file to bean + Users usersBean; + try { + usersBean = (Users) xmlParseBase.baseParseXmlToBean(path); + } catch (Exception e) { + LOGGER.warn("parseXmlToBean Exception", e); + throw e; + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Xml to Shardings is :" + usersBean); + } + return userBeanToJson(usersBean); + } + + private void userListToMap(List userList, Map blackListMap, ProblemReporter problemReporter) { + for (Object userObj : userList) { + User user = (User) userObj; + String name = user.getName(); + String password = user.getPassword(); + String usingDecryptStr = Optional.ofNullable(user.getUsingDecrypt()).orElse("false"); + final String whiteIPsStr = user.getWhiteIPs(); + final int maxCon = Optional.ofNullable(user.getMaxCon()).orElse(-1); + boolean usingDecrypt; + + if (StringUtil.isEmpty(name)) { + throw new ConfigException("one of users' name is empty"); + } + if (StringUtil.isEmpty(password)) { + throw new ConfigException("password of " + name + " is empty"); + } + usingDecrypt = Boolean.parseBoolean(usingDecryptStr); + password = DecryptUtil.decrypt(usingDecrypt, name, password); + IPAddressUtil.checkWhiteIPs(whiteIPsStr); + UserConfig userConfig = new UserConfig(name, password, usingDecrypt, whiteIPsStr, String.valueOf(maxCon)); + + if (user instanceof ManagerUser) { + fillManagerUser(userConfig, (ManagerUser) user); + } else if (user instanceof ShardingUser) { + fillShardingUser(userConfig, (ShardingUser) user, blackListMap, problemReporter); + } else if (user instanceof RwSplitUser) { + fillRwSplitUser(userConfig, (RwSplitUser) user, blackListMap, problemReporter); + } + } + } + + private void fillRwSplitUser(UserConfig userConfig, RwSplitUser rwSplitUser, Map blackListMap, ProblemReporter problemReporter) { + String tenant = rwSplitUser.getTenant(); + String dbGroup = rwSplitUser.getDbGroup(); + String blacklistStr = rwSplitUser.getBlacklist(); + + UserName userName = new UserName(userConfig.getName(), tenant); + if (this.userConfigMap.containsKey(userName)) { + throw new ConfigException("User [" + userName + "] has already existed"); + } + if (StringUtil.isEmpty(dbGroup)) { + throw new ConfigException("User [" + userName + "]'s dbGroup is empty"); + } + + WallProvider wallProvider = null; + if (!StringUtil.isEmpty(blacklistStr)) { + wallProvider = blackListMap.get(blacklistStr); + if (wallProvider == null) { + problemReporter.warn("blacklist[" + blacklistStr + "] for user [" + userName + "] is not found, it will be ignore"); + } else { + wallProvider.setName(blacklistStr); + } + } + + RwSplitUserConfig rwSplitUserConfig = new RwSplitUserConfig(userConfig, userName.getTenant(), wallProvider, dbGroup); + rwSplitUserConfig.setId(this.userId.incrementAndGet()); + this.userConfigMap.put(userName, rwSplitUserConfig); + } + + private void fillShardingUser(UserConfig userConfig, ShardingUser shardingUser, Map blackListMap, ProblemReporter problemReporter) { + String tenant = shardingUser.getTenant(); + final boolean readOnly = Optional.ofNullable(shardingUser.getReadOnly()).orElse(false); + String schemas = shardingUser.getSchemas(); + String blacklistStr = shardingUser.getBlacklist(); + + UserName userName = new UserName(userConfig.getName(), tenant); + if (this.userConfigMap.containsKey(userName)) { + throw new ConfigException("User [" + userName + "] has already existed"); + } + if (StringUtil.isEmpty(schemas)) { + throw new ConfigException("User [" + userName + "]'s schemas is empty"); + } + String[] strArray = SplitUtil.split(schemas, ',', true); + + WallProvider wallProvider = null; + if (!StringUtil.isEmpty(blacklistStr)) { + wallProvider = blackListMap.get(blacklistStr); + if (wallProvider == null) { + problemReporter.warn("blacklist[" + blacklistStr + "] for user [" + userName + "] is not found, it will be ignore"); + } else { + wallProvider.setName(blacklistStr); + } + } + // load DML Privileges + Privileges privileges = shardingUser.getPrivileges(); + UserPrivilegesConfig privilegesConfig = loadPrivilegesConfig(privileges, userConfig); + + ShardingUserConfig shardingUserConfig = new ShardingUserConfig(userConfig, userName.getTenant(), wallProvider, readOnly, new HashSet<>(Arrays.asList(strArray)), privilegesConfig); + shardingUserConfig.setId(this.userId.incrementAndGet()); + this.userConfigMap.put(userName, shardingUserConfig); + } + + private UserPrivilegesConfig loadPrivilegesConfig(Privileges privileges, UserConfig userConfig) { + List schemaList = null == privileges ? Lists.newArrayList() : Optional.ofNullable(privileges.getSchema()).orElse(Collections.EMPTY_LIST); + boolean check = null == privileges ? false : Optional.ofNullable(privileges.getCheck()).orElse(false); + if (schemaList.isEmpty()) { + return null; + } + UserPrivilegesConfig privilegesConfig = new UserPrivilegesConfig(); + privilegesConfig.setCheck(check); + for (Schema schema : schemaList) { + String schemaName = schema.getName(); + String schemaDml = schema.getDml(); + + if (!XMLUserLoader.DML_PATTERN.matcher(schemaDml).matches()) + throw new ConfigException("User [" + userConfig.getName() + "]'s schema [" + schemaName + "]'s privilege's dml is not correct"); + int[] dml1Array = new int[schemaDml.length()]; + for (int offset1 = 0; offset1 < schemaDml.length(); offset1++) { + dml1Array[offset1] = Character.getNumericValue(schemaDml.charAt(offset1)); + } + UserPrivilegesConfig.SchemaPrivilege schemaPrivilege = new UserPrivilegesConfig.SchemaPrivilege(); + schemaPrivilege.setDml(dml1Array); + + List tableList = schema.getTable(); + for (Table table : tableList) { + String tableDml = table.getDml(); + UserPrivilegesConfig.TablePrivilege tablePrivilege = new UserPrivilegesConfig.TablePrivilege(); + + if (!XMLUserLoader.DML_PATTERN.matcher(tableDml).matches()) + throw new ConfigException("User [" + userConfig.getName() + "]'s schema [" + schemaName + "]'s table [" + tableDml + "]'s privilege's dml is not correct"); + int[] dml2Array = new int[tableDml.length()]; + for (int offset2 = 0; offset2 < tableDml.length(); offset2++) { + dml2Array[offset2] = Character.getNumericValue(tableDml.charAt(offset2)); + } + tablePrivilege.setDml(dml2Array); + schemaPrivilege.addTablePrivilege(table.getName(), tablePrivilege); + } + privilegesConfig.addSchemaPrivilege(schemaName, schemaPrivilege); + } + return privilegesConfig; + } + + private void fillManagerUser(UserConfig userConfig, ManagerUser managerUser) { + Boolean readOnly = Optional.ofNullable(managerUser.getReadOnly()).orElse(false); + UserName userName = new UserName(userConfig.getName()); + if (this.userConfigMap.containsKey(userName)) { + throw new ConfigException("User [name:" + userConfig.getName() + "] has already existed"); + } + ManagerUserConfig managerUserConfig = new ManagerUserConfig(userConfig, readOnly); + managerUserConfig.setId(this.userId.incrementAndGet()); + + this.userConfigMap.put(userName, managerUserConfig); + } + + private Map blackListToMap(List blacklist, ProblemReporter problemReporter) throws InvocationTargetException, IllegalAccessException { + Map blackListMap = Maps.newLinkedHashMap(); + for (BlackList blackList : blacklist) { + String name = blackList.getName(); + List propertyList = blackList.getProperty(); + if (blackListMap.containsKey(name)) { + throw new ConfigException("blacklist[" + name + "] has already existed"); + } + + Properties props2 = new Properties(); + Properties props = new Properties(); + propertyList.forEach(property -> props.put(property.getName(), property.getValue())); + props2.putAll(props); + this.blackListConfigMap.put(name, props2); + + WallConfig wallConfig = new WallConfig(); + ParameterMapping.mapping(wallConfig, props, problemReporter); + if (props.size() > 0) { + String[] propItem = new String[props.size()]; + props.keySet().toArray(propItem); + throw new ConfigException("blacklist item(s) is not recognized: " + StringUtil.join(propItem, ",")); + } + WallProvider provider = new MySqlWallProvider(wallConfig); + provider.setBlackListEnable(true); + blackListMap.put(name, provider); + } + return blackListMap; + } + + public Map getUserConfigMap() { + return userConfigMap; + } + + public Map getBlackListConfigMap() { + return blackListConfigMap; + } +} diff --git a/src/main/java/com/actiontech/dble/config/loader/xml/XMLDbLoader.java b/src/main/java/com/actiontech/dble/config/loader/xml/XMLDbLoader.java index a58356cb21..4d20ffee11 100644 --- a/src/main/java/com/actiontech/dble/config/loader/xml/XMLDbLoader.java +++ b/src/main/java/com/actiontech/dble/config/loader/xml/XMLDbLoader.java @@ -38,7 +38,7 @@ public class XMLDbLoader { private final Map dbGroupConfigs; private ProblemReporter problemReporter; private final Map dbGroups; - private static final Pattern PATTERN_DB = Pattern.compile("([" + DB_NAME_FORMAT + "]+)", Pattern.CASE_INSENSITIVE); + public static final Pattern PATTERN_DB = Pattern.compile("([" + DB_NAME_FORMAT + "]+)", Pattern.CASE_INSENSITIVE); public XMLDbLoader(String dbFile, ProblemReporter problemReporter) { this.dbGroupConfigs = new LinkedHashMap<>(); @@ -243,11 +243,11 @@ private Map initDbGroups(Map nod return nodes; } - private PhysicalDbInstance createDbInstance(DbGroupConfig conf, DbInstanceConfig node, boolean isRead) { + public static PhysicalDbInstance createDbInstance(DbGroupConfig conf, DbInstanceConfig node, boolean isRead) { return new MySQLInstance(node, conf, isRead); } - private PhysicalDbGroup getPhysicalDBPoolSingleWH(DbGroupConfig conf) { + public static PhysicalDbGroup getPhysicalDBPoolSingleWH(DbGroupConfig conf) { //create PhysicalDbInstance for writeDirectly host PhysicalDbInstance writeSource = createDbInstance(conf, conf.getWriteInstanceConfig(), false); PhysicalDbInstance[] readSources = new PhysicalDbInstance[conf.getReadInstanceConfigs().length]; diff --git a/src/main/java/com/actiontech/dble/config/loader/xml/XMLShardingLoader.java b/src/main/java/com/actiontech/dble/config/loader/xml/XMLShardingLoader.java index 87cd1fc933..5e9d70c5bd 100644 --- a/src/main/java/com/actiontech/dble/config/loader/xml/XMLShardingLoader.java +++ b/src/main/java/com/actiontech/dble/config/loader/xml/XMLShardingLoader.java @@ -166,7 +166,7 @@ private void loadSchemas(Element root) { makeAllErRelations(); } - private int getSqlMaxLimit(String sqlMaxLimitStr, int defaultMaxLimit) { + public static int getSqlMaxLimit(String sqlMaxLimitStr, int defaultMaxLimit) { // sql result size limit if (sqlMaxLimitStr != null && !sqlMaxLimitStr.isEmpty()) { defaultMaxLimit = Integer.parseInt(sqlMaxLimitStr); @@ -536,7 +536,7 @@ private void loadShardingNode(Element root) { } } - private List mergerHostDatabase(String[] hostStrings, String[] databases) { + public static List mergerHostDatabase(String[] hostStrings, String[] databases) { List mhdList = new ArrayList<>(); for (String hostString : hostStrings) { for (String database : databases) { @@ -611,7 +611,7 @@ private void setFunctionAlias() { } } - private AbstractPartitionAlgorithm createFunction(String name, String clazz) + public static AbstractPartitionAlgorithm createFunction(String name, String clazz) throws ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException { diff --git a/src/main/java/com/actiontech/dble/config/loader/xml/XMLUserLoader.java b/src/main/java/com/actiontech/dble/config/loader/xml/XMLUserLoader.java index b64c56d703..dd146602e9 100644 --- a/src/main/java/com/actiontech/dble/config/loader/xml/XMLUserLoader.java +++ b/src/main/java/com/actiontech/dble/config/loader/xml/XMLUserLoader.java @@ -43,7 +43,7 @@ public class XMLUserLoader { private static final String DEFAULT_XML = "/" + ConfigFileName.USER_XML; private ProblemReporter problemReporter; private AtomicInteger userId = new AtomicInteger(0); - private static final Pattern DML_PATTERN = Pattern.compile("^[0|1]{4}$"); + public static final Pattern DML_PATTERN = Pattern.compile("^[0|1]{4}$"); private Document document; // whether db.xml contains shardingUser private boolean containsShardingUser; diff --git a/src/main/java/com/actiontech/dble/config/model/sharding/SchemaConfig.java b/src/main/java/com/actiontech/dble/config/model/sharding/SchemaConfig.java index dcd6af3645..c6fff1c728 100644 --- a/src/main/java/com/actiontech/dble/config/model/sharding/SchemaConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/sharding/SchemaConfig.java @@ -8,6 +8,7 @@ import com.actiontech.dble.config.model.sharding.table.ChildTableConfig; import com.actiontech.dble.config.model.sharding.table.ERTable; import com.actiontech.dble.config.model.sharding.table.ShardingTableConfig; +import com.actiontech.dble.util.StringUtil; import java.util.*; @@ -234,4 +235,22 @@ public Map> getFuncNodeERMap() { return funcNodeERMap; } + + public boolean equalsBaseInfo(SchemaConfig schemaConfig) { + return StringUtil.equalsWithEmpty(this.name, schemaConfig.getName()) && + this.noSharding == schemaConfig.isNoSharding() && + StringUtil.equalsWithEmpty(this.shardingNode, schemaConfig.getShardingNode()) && + StringUtil.equalsWithEmpty(this.metaShardingNode, schemaConfig.getMetaShardingNode()) && + isEquals(this.allShardingNodes, schemaConfig.getAllShardingNodes()); + + } + + + private boolean isEquals(Set o1, Set o2) { + if (o1 == null) { + return o2 == null; + } + return o1 == o2 || o1.equals(o2); + } + } diff --git a/src/main/java/com/actiontech/dble/config/model/sharding/table/BaseTableConfig.java b/src/main/java/com/actiontech/dble/config/model/sharding/table/BaseTableConfig.java index f7dff4301f..249223fe8d 100644 --- a/src/main/java/com/actiontech/dble/config/model/sharding/table/BaseTableConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/sharding/table/BaseTableConfig.java @@ -5,6 +5,8 @@ package com.actiontech.dble.config.model.sharding.table; +import com.actiontech.dble.util.StringUtil; + import java.util.List; public abstract class BaseTableConfig { @@ -42,4 +44,18 @@ public List getShardingNodes() { } public abstract BaseTableConfig lowerCaseCopy(BaseTableConfig parent); + + + public boolean equalsBaseInfo(BaseTableConfig baseTableConfig) { + return StringUtil.equalsWithEmpty(this.name, baseTableConfig.getName()) && + this.maxLimit == baseTableConfig.getMaxLimit() && + isEquals(this.shardingNodes, baseTableConfig.getShardingNodes()); + } + + private boolean isEquals(List o1, List o2) { + if (o1 == null) { + return o2 == null; + } + return o1 == o2 || o1.equals(o2); + } } diff --git a/src/main/java/com/actiontech/dble/config/model/sharding/table/ChildTableConfig.java b/src/main/java/com/actiontech/dble/config/model/sharding/table/ChildTableConfig.java index 4041258489..f25f7ea879 100644 --- a/src/main/java/com/actiontech/dble/config/model/sharding/table/ChildTableConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/sharding/table/ChildTableConfig.java @@ -5,6 +5,8 @@ package com.actiontech.dble.config.model.sharding.table; +import com.actiontech.dble.util.StringUtil; + import java.util.List; public class ChildTableConfig extends BaseTableConfig { @@ -129,4 +131,22 @@ public ShardingTableConfig getDirectRouteTC() { return directRouteTC; } + + public boolean equalsBaseInfo(ChildTableConfig childTableConfig) { + return super.equalsBaseInfo(childTableConfig) && + this.parentTC.equalsBaseInfo(childTableConfig.getParentTC()) && + StringUtil.equalsWithEmpty(this.joinColumn, childTableConfig.getJoinColumn()) && + StringUtil.equalsWithEmpty(this.parentColumn, childTableConfig.getParentColumn()) && + StringUtil.equalsWithEmpty(this.incrementColumn, childTableConfig.getIncrementColumn()) && + StringUtil.equalsWithEmpty(this.locateRTableKeySql, childTableConfig.getLocateRTableKeySql()) && + isEquals(this.directRouteTC, childTableConfig.getDirectRouteTC()); + } + + private boolean isEquals(ShardingTableConfig o1, ShardingTableConfig o2) { + if (o1 == null) { + return o2 == null; + } + return o1 == o2 || o1.equalsBaseInfo(o2); + } + } diff --git a/src/main/java/com/actiontech/dble/config/model/sharding/table/GlobalTableConfig.java b/src/main/java/com/actiontech/dble/config/model/sharding/table/GlobalTableConfig.java index c0974fe210..3ede62d5f1 100644 --- a/src/main/java/com/actiontech/dble/config/model/sharding/table/GlobalTableConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/sharding/table/GlobalTableConfig.java @@ -5,6 +5,8 @@ package com.actiontech.dble.config.model.sharding.table; +import com.actiontech.dble.util.StringUtil; + import java.util.List; public class GlobalTableConfig extends BaseTableConfig { @@ -36,4 +38,12 @@ public String getCheckClass() { public BaseTableConfig lowerCaseCopy(BaseTableConfig parent) { return new GlobalTableConfig(this.name.toLowerCase(), this.maxLimit, this.shardingNodes, this.cron, this.checkClass, this.globalCheck); } + + + public boolean equalsBaseInfo(GlobalTableConfig globalTableConfig) { + return super.equalsBaseInfo(globalTableConfig) && + this.globalCheck == globalTableConfig.isGlobalCheck() && + StringUtil.equalsWithEmpty(this.cron, globalTableConfig.getCron()) && + StringUtil.equalsWithEmpty(this.checkClass, globalTableConfig.getCheckClass()); + } } diff --git a/src/main/java/com/actiontech/dble/config/model/sharding/table/ShardingTableConfig.java b/src/main/java/com/actiontech/dble/config/model/sharding/table/ShardingTableConfig.java index 234c6172e3..6b286468b7 100644 --- a/src/main/java/com/actiontech/dble/config/model/sharding/table/ShardingTableConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/sharding/table/ShardingTableConfig.java @@ -6,6 +6,7 @@ package com.actiontech.dble.config.model.sharding.table; import com.actiontech.dble.route.function.AbstractPartitionAlgorithm; +import com.actiontech.dble.util.StringUtil; import java.util.List; @@ -16,7 +17,7 @@ public class ShardingTableConfig extends BaseTableConfig { private final boolean sqlRequiredSharding; public ShardingTableConfig(String name, int maxLimit, List shardingNodes, String incrementColumn, - AbstractPartitionAlgorithm function, String shardingColumn, boolean sqlRequiredSharding) { + AbstractPartitionAlgorithm function, String shardingColumn, boolean sqlRequiredSharding) { super(name, maxLimit, shardingNodes); this.incrementColumn = incrementColumn; this.function = function; @@ -46,4 +47,13 @@ public BaseTableConfig lowerCaseCopy(BaseTableConfig parent) { return new ShardingTableConfig(this.name.toLowerCase(), this.maxLimit, this.shardingNodes, this.incrementColumn, this.function, this.shardingColumn, this.sqlRequiredSharding); } + + + public boolean equalsBaseInfo(ShardingTableConfig shardingTableConfig) { + return super.equalsBaseInfo(shardingTableConfig) && + StringUtil.equalsWithEmpty(this.incrementColumn, shardingTableConfig.getIncrementColumn()) && + this.function.equals(shardingTableConfig.getFunction()) && + StringUtil.equalsWithEmpty(this.shardingColumn, shardingTableConfig.getShardingColumn()) && + this.sqlRequiredSharding == shardingTableConfig.isSqlRequiredSharding(); + } } diff --git a/src/main/java/com/actiontech/dble/config/model/user/ManagerUserConfig.java b/src/main/java/com/actiontech/dble/config/model/user/ManagerUserConfig.java index 50b159bfcb..12b1725d3d 100644 --- a/src/main/java/com/actiontech/dble/config/model/user/ManagerUserConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/user/ManagerUserConfig.java @@ -42,4 +42,9 @@ public void isValidSchemaInfo(UserName user, SchemaUtil.SchemaInfo schemaInfo) t throw new SQLException(msg, "42S02", ErrorCode.ER_NO_SUCH_TABLE); } } + + public boolean equalsBaseInfo(ManagerUserConfig managerUserConfig) { + return super.equalsBaseInfo(managerUserConfig) && + this.readOnly == managerUserConfig.isReadOnly(); + } } diff --git a/src/main/java/com/actiontech/dble/config/model/user/RwSplitUserConfig.java b/src/main/java/com/actiontech/dble/config/model/user/RwSplitUserConfig.java index 3130365ee8..b067920c0f 100644 --- a/src/main/java/com/actiontech/dble/config/model/user/RwSplitUserConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/user/RwSplitUserConfig.java @@ -5,6 +5,7 @@ package com.actiontech.dble.config.model.user; +import com.actiontech.dble.util.StringUtil; import com.alibaba.druid.wall.WallProvider; public class RwSplitUserConfig extends ServerUserConfig { @@ -20,4 +21,9 @@ public String getDbGroup() { } + public boolean equalsBaseInfo(RwSplitUserConfig rwSplitUserConfig) { + return super.equalsBaseInfo(rwSplitUserConfig) && + StringUtil.equalsWithEmpty(this.dbGroup, rwSplitUserConfig.getDbGroup()); + } + } diff --git a/src/main/java/com/actiontech/dble/config/model/user/ServerUserConfig.java b/src/main/java/com/actiontech/dble/config/model/user/ServerUserConfig.java index 60feb18bb5..a64655bf70 100644 --- a/src/main/java/com/actiontech/dble/config/model/user/ServerUserConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/user/ServerUserConfig.java @@ -5,6 +5,7 @@ package com.actiontech.dble.config.model.user; +import com.actiontech.dble.util.StringUtil; import com.alibaba.druid.wall.WallProvider; public abstract class ServerUserConfig extends UserConfig { @@ -25,4 +26,22 @@ public WallProvider getBlacklist() { return blacklist; } + + public boolean equalsBaseInfo(ServerUserConfig serverUserConfig) { + return super.equalsBaseInfo(serverUserConfig) && + StringUtil.equalsWithEmpty(this.tenant, serverUserConfig.getTenant()) && + isEquals(this.blacklist, serverUserConfig.getBlacklist()); + } + + + private boolean isEquals(WallProvider o1, WallProvider o2) { + if (o1 == null || o2 == null) { + return o1 == o2; + } + if (o1.getAttributes() == null || o2.getAttributes() == null) { + return o1.getAttributes() == o2.getAttributes(); + } + return o1 == o2 || o1.getAttributes().equals(o2.getAttributes()); + } + } diff --git a/src/main/java/com/actiontech/dble/config/model/user/ShardingUserConfig.java b/src/main/java/com/actiontech/dble/config/model/user/ShardingUserConfig.java index 5910637ec5..14a87ca880 100644 --- a/src/main/java/com/actiontech/dble/config/model/user/ShardingUserConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/user/ShardingUserConfig.java @@ -65,4 +65,18 @@ public void isValidSchemaInfo(UserName user, SchemaUtil.SchemaInfo schemaInfo) t schemaInfo.setSchemaConfig(schemaConfig); } } + + public boolean equalsBaseInfo(ShardingUserConfig shardingUserConfig) { + return super.equalsBaseInfo(shardingUserConfig) && + this.readOnly == shardingUserConfig.isReadOnly() && + this.schemas.equals(shardingUserConfig.getSchemas()) && + isEquals(this.privilegesConfig, shardingUserConfig.getPrivilegesConfig()); + } + + private boolean isEquals(UserPrivilegesConfig o1, UserPrivilegesConfig o2) { + if (o1 == null) { + return o2 == null; + } + return o1 == o2 || o1.equalsBaseInfo(o2); + } } diff --git a/src/main/java/com/actiontech/dble/config/model/user/UserConfig.java b/src/main/java/com/actiontech/dble/config/model/user/UserConfig.java index fe165517b4..280bc184aa 100644 --- a/src/main/java/com/actiontech/dble/config/model/user/UserConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/user/UserConfig.java @@ -86,4 +86,12 @@ public int getMaxCon() { public void isValidSchemaInfo(UserName user, SchemaUtil.SchemaInfo schemaInfo) throws SQLException { } + + public boolean equalsBaseInfo(UserConfig userConfig) { + return StringUtil.equalsWithEmpty(this.name, userConfig.getName()) && + StringUtil.equalsWithEmpty(this.password, userConfig.getPassword()) && + this.isEncrypt == userConfig.isEncrypt() && + this.maxCon == userConfig.getMaxCon() && + this.whiteIPs.equals(userConfig.getWhiteIPs()); + } } diff --git a/src/main/java/com/actiontech/dble/config/model/user/UserPrivilegesConfig.java b/src/main/java/com/actiontech/dble/config/model/user/UserPrivilegesConfig.java index db1aa096d4..5f32ff2c6d 100644 --- a/src/main/java/com/actiontech/dble/config/model/user/UserPrivilegesConfig.java +++ b/src/main/java/com/actiontech/dble/config/model/user/UserPrivilegesConfig.java @@ -5,6 +5,7 @@ package com.actiontech.dble.config.model.user; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -51,6 +52,15 @@ public void changeMapToLowerCase() { schemaPrivileges = newSchemaPrivileges; } + + public boolean equalsBaseInfo(UserPrivilegesConfig userPrivilegesConfig) { + boolean equalTableInfo1 = this.schemaPrivileges.entrySet().stream().allMatch(schemaPrivilegeEntry -> null != userPrivilegesConfig.getSchemaPrivilege(schemaPrivilegeEntry.getKey()) && userPrivilegesConfig.getSchemaPrivilege(schemaPrivilegeEntry.getKey()).equalsBaseInfo(schemaPrivilegeEntry.getValue())); + boolean equalTableInfo2 = userPrivilegesConfig.getSchemaPrivileges().entrySet().stream().allMatch(schemaPrivilegeEntry -> null != this.schemaPrivileges.get(schemaPrivilegeEntry.getKey()) && this.schemaPrivileges.get(schemaPrivilegeEntry.getKey()).equalsBaseInfo(schemaPrivilegeEntry.getValue())); + return this.check == userPrivilegesConfig.isCheck() && + equalTableInfo1 && + equalTableInfo2; + } + public static class SchemaPrivilege { private int[] dml = new int[]{0, 0, 0, 0}; @@ -89,6 +99,15 @@ public TablePrivilege getTablePrivilege(String tableName) { public Set getTables() { return tablePrivileges.keySet(); } + + + public boolean equalsBaseInfo(SchemaPrivilege schemaPrivilege) { + boolean equalTableInfo1 = this.tablePrivileges.entrySet().stream().allMatch(tablePrivilegeEntry -> null != schemaPrivilege.getTablePrivilege(tablePrivilegeEntry.getKey()) && schemaPrivilege.getTablePrivilege(tablePrivilegeEntry.getKey()).equalsBaseInfo(tablePrivilegeEntry.getValue())); + boolean equalTableInfo2 = schemaPrivilege.getTablePrivileges().entrySet().stream().allMatch(tablePrivilegeEntry -> null != this.tablePrivileges.get(tablePrivilegeEntry.getKey()) && this.tablePrivileges.get(tablePrivilegeEntry.getKey()).equalsBaseInfo(tablePrivilegeEntry.getValue())); + return Arrays.equals(this.dml, schemaPrivilege.getDml()) && + equalTableInfo1 && + equalTableInfo2; + } } public static class TablePrivilege { @@ -101,5 +120,9 @@ public int[] getDml() { public void setDml(int[] dml) { this.dml = dml; } + + public boolean equalsBaseInfo(TablePrivilege tablePrivilege) { + return Arrays.equals(this.dml, tablePrivilege.getDml()); + } } } diff --git a/src/main/java/com/actiontech/dble/config/util/DbXmlWriteJob.java b/src/main/java/com/actiontech/dble/config/util/DbXmlWriteJob.java index ab2d235f1d..aeec3afa19 100644 --- a/src/main/java/com/actiontech/dble/config/util/DbXmlWriteJob.java +++ b/src/main/java/com/actiontech/dble/config/util/DbXmlWriteJob.java @@ -1,11 +1,13 @@ package com.actiontech.dble.config.util; +import com.actiontech.dble.DbleServer; import com.actiontech.dble.backend.datasource.PhysicalDbGroup; import com.actiontech.dble.backend.datasource.PhysicalDbInstance; import com.actiontech.dble.cluster.ClusterLogic; import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBGroup; import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBInstance; +import com.actiontech.dble.config.converter.DBConverter; import com.actiontech.dble.singleton.HaConfigManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +51,7 @@ public void run() { } } HaConfigManager.getInstance().write(dbGroups, reloadIndex); + DbleServer.getInstance().getConfig().setDbConfig(DBConverter.dbBeanToJson(dbGroups)); } catch (Exception e) { errorMessage = e.getMessage(); HaConfigManager.getInstance().log("get error from SchemaWriteJob", e); diff --git a/src/main/java/com/actiontech/dble/route/function/PartitionByJumpConsistentHash.java b/src/main/java/com/actiontech/dble/route/function/PartitionByJumpConsistentHash.java index c34d029eb2..15a05c9edb 100644 --- a/src/main/java/com/actiontech/dble/route/function/PartitionByJumpConsistentHash.java +++ b/src/main/java/com/actiontech/dble/route/function/PartitionByJumpConsistentHash.java @@ -108,4 +108,30 @@ public void setPartitionCount(int totalBuckets) { public int getPartitionCount() { return partitionCount; } + + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PartitionByJumpConsistentHash other = (PartitionByJumpConsistentHash) o; + return other.partitionCount == this.partitionCount && + other.hashSliceStart == this.hashSliceStart && + other.hashSliceEnd == this.hashSliceEnd; + } + + @Override + public int hashCode() { + int hashCode = -1; + if (partitionCount != 0) { + hashCode *= partitionCount; + } + if (hashSliceEnd - hashSliceStart != 0) { + hashCode *= (hashSliceEnd - hashSliceStart); + } + return hashCode; + } + } diff --git a/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceMySQLHandler.java b/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceMySQLHandler.java index 8eeebbb54f..6179b6c8e2 100644 --- a/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceMySQLHandler.java +++ b/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceMySQLHandler.java @@ -6,6 +6,7 @@ package com.actiontech.dble.route.sequence.handler; import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.converter.SequenceConverter; import com.actiontech.dble.config.util.ConfigException; import com.actiontech.dble.route.util.PropertiesUtil; import org.slf4j.Logger; @@ -30,6 +31,15 @@ public void load(boolean isLowerCaseTableNames) { putNewSequenceVals(props); } + @Override + public void loadByJson(boolean isLowerCaseTableNames, String sequenceJson) { + SequenceConverter sequenceConverter = new SequenceConverter(); + Properties props = sequenceConverter.jsonToProperties(sequenceJson); + props = PropertiesUtil.handleLowerCase(props, isLowerCaseTableNames); + removeDesertedSequenceVals(props); + putNewSequenceVals(props); + } + public Set getShardingNodes() { return shardingNodes; } diff --git a/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceZKHandler.java b/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceZKHandler.java index 146ffbc61d..e18f083c49 100644 --- a/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceZKHandler.java +++ b/src/main/java/com/actiontech/dble/route/sequence/handler/IncrSequenceZKHandler.java @@ -7,6 +7,7 @@ import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.converter.SequenceConverter; import com.actiontech.dble.config.model.ClusterConfig; import com.actiontech.dble.route.util.PropertiesUtil; import com.actiontech.dble.util.KVPathUtil; @@ -50,20 +51,36 @@ public class IncrSequenceZKHandler extends IncrSequenceHandler { private Properties props; public void load(boolean isLowerCaseTableNames) { - props = PropertiesUtil.loadProps(ConfigFileName.SEQUENCE_FILE_NAME, isLowerCaseTableNames); + this.props = PropertiesUtil.loadProps(ConfigFileName.SEQUENCE_FILE_NAME, isLowerCaseTableNames); String zkAddress = ClusterConfig.getInstance().getClusterIP(); if (zkAddress == null) { throw new RuntimeException("please check ClusterIP is correct in config file \"cluster.cnf\" ."); } try { - initializeZK(props, zkAddress); + initializeZK(this.props, zkAddress); + } catch (Exception e) { + LOGGER.warn("Error caught while initializing ZK:" + e.getCause()); + } + } + + @Override + public void loadByJson(boolean isLowerCaseTableNames, String sequenceJson) { + SequenceConverter sequenceConverter = new SequenceConverter(); + this.props = sequenceConverter.jsonToProperties(sequenceJson); + this.props = PropertiesUtil.handleLowerCase(this.props, isLowerCaseTableNames); + String zkAddress = ClusterConfig.getInstance().getClusterIP(); + if (zkAddress == null) { + throw new RuntimeException("please check ClusterIP is correct in config file \"cluster.cnf\" ."); + } + try { + initializeZK(this.props, zkAddress); } catch (Exception e) { LOGGER.warn("Error caught while initializing ZK:" + e.getCause()); } } private void threadLocalLoad() throws Exception { - Enumeration enu = props.propertyNames(); + Enumeration enu = this.props.propertyNames(); while (enu.hasMoreElements()) { String key = (String) enu.nextElement(); if (key.endsWith(KEY_MIN_NAME)) { @@ -103,11 +120,11 @@ private void handle(String key) throws Exception { Stat stat = this.client.checkExists().forPath(seqPath); if (stat == null || (stat.getDataLength() == 0)) { - paraValMap.put(table + KEY_MIN_NAME, props.getProperty(key)); - paraValMap.put(table + KEY_MAX_NAME, props.getProperty(table + KEY_MAX_NAME)); - paraValMap.put(table + KEY_CUR_NAME, props.getProperty(table + KEY_CUR_NAME)); + paraValMap.put(table + KEY_MIN_NAME, this.props.getProperty(key)); + paraValMap.put(table + KEY_MAX_NAME, this.props.getProperty(table + KEY_MAX_NAME)); + paraValMap.put(table + KEY_CUR_NAME, this.props.getProperty(table + KEY_CUR_NAME)); try { - String val = props.getProperty(table + KEY_MIN_NAME); + String val = this.props.getProperty(table + KEY_MIN_NAME); client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(PATH + table + SEQ, val.getBytes()); } catch (Exception e) { LOGGER.debug("Node exists! Maybe other instance is initializing!"); @@ -150,13 +167,13 @@ public Boolean fetchNextPeriod(String prefixName) { } if (paraValMap.get(prefixName + KEY_MAX_NAME) == null) { - paraValMap.put(prefixName + KEY_MAX_NAME, props.getProperty(prefixName + KEY_MAX_NAME)); + paraValMap.put(prefixName + KEY_MAX_NAME, this.props.getProperty(prefixName + KEY_MAX_NAME)); } if (paraValMap.get(prefixName + KEY_MIN_NAME) == null) { - paraValMap.put(prefixName + KEY_MIN_NAME, props.getProperty(prefixName + KEY_MIN_NAME)); + paraValMap.put(prefixName + KEY_MIN_NAME, this.props.getProperty(prefixName + KEY_MIN_NAME)); } if (paraValMap.get(prefixName + KEY_CUR_NAME) == null) { - paraValMap.put(prefixName + KEY_CUR_NAME, props.getProperty(prefixName + KEY_CUR_NAME)); + paraValMap.put(prefixName + KEY_CUR_NAME, this.props.getProperty(prefixName + KEY_CUR_NAME)); } long period = Long.parseLong(paraValMap.get(prefixName + KEY_MAX_NAME)) - Long.parseLong(paraValMap.get(prefixName + KEY_MIN_NAME)); diff --git a/src/main/java/com/actiontech/dble/route/sequence/handler/SequenceHandler.java b/src/main/java/com/actiontech/dble/route/sequence/handler/SequenceHandler.java index 4cf1aca866..0e1026b190 100644 --- a/src/main/java/com/actiontech/dble/route/sequence/handler/SequenceHandler.java +++ b/src/main/java/com/actiontech/dble/route/sequence/handler/SequenceHandler.java @@ -18,4 +18,7 @@ public interface SequenceHandler { void load(boolean isLowerCaseTableNames); + default void loadByJson(boolean isLowerCaseTableNames, String sequenceJson) { + } + } diff --git a/src/main/java/com/actiontech/dble/route/util/PropertiesUtil.java b/src/main/java/com/actiontech/dble/route/util/PropertiesUtil.java index c9acf1068a..0c551fd8ee 100644 --- a/src/main/java/com/actiontech/dble/route/util/PropertiesUtil.java +++ b/src/main/java/com/actiontech/dble/route/util/PropertiesUtil.java @@ -5,11 +5,11 @@ package com.actiontech.dble.route.util; +import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.route.sequence.handler.IncrSequenceHandler; import com.actiontech.dble.util.ResourceUtil; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.util.Enumeration; import java.util.Properties; @@ -39,6 +39,11 @@ public static Properties loadProps(String propsFile) { public static Properties loadProps(String propsFile, boolean isLowerCaseTableNames) { Properties props = loadProps(propsFile); + return handleLowerCase(props, isLowerCaseTableNames); + } + + + public static Properties handleLowerCase(Properties props, boolean isLowerCaseTableNames) { if (isLowerCaseTableNames) { Properties newProps = new Properties(); Enumeration enu = props.propertyNames(); @@ -57,4 +62,13 @@ public static Properties loadProps(String propsFile, boolean isLowerCaseTableNam return props; } } + + + public static void storeProps(Properties props, String propsFile) { + try (OutputStream os = new FileOutputStream(new File(ResourceUtil.getResourcePathFromRoot(ClusterPathUtil.LOCAL_WRITE_PATH)).getPath() + File.separator + propsFile)) { + props.store(os, "\n Copyright (C) 2016-2020 ActionTech.\n License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.\n"); + } catch (IOException e) { + throw new java.lang.RuntimeException(e); + } + } } diff --git a/src/main/java/com/actiontech/dble/services/manager/handler/DeleteHandler.java b/src/main/java/com/actiontech/dble/services/manager/handler/DeleteHandler.java index c4e9fcb92d..7114860462 100644 --- a/src/main/java/com/actiontech/dble/services/manager/handler/DeleteHandler.java +++ b/src/main/java/com/actiontech/dble/services/manager/handler/DeleteHandler.java @@ -5,8 +5,13 @@ package com.actiontech.dble.services.manager.handler; +import com.actiontech.dble.cluster.ClusterHelper; +import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.DistributeLock; import com.actiontech.dble.cluster.values.ConfStatus; import com.actiontech.dble.config.ErrorCode; +import com.actiontech.dble.config.model.ClusterConfig; +import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.config.util.ConfigException; import com.actiontech.dble.net.mysql.OkPacket; import com.actiontech.dble.net.mysql.RowDataPacket; @@ -26,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.SQLException; import java.util.LinkedHashMap; import java.util.List; @@ -87,6 +91,16 @@ public void handle(String stmt, ManagerService service) { service.writeErrMessage("42000", "Access denied for table '" + managerBaseTable.getTableName() + "'", ErrorCode.ER_ACCESS_DENIED_ERROR); return; } + + DistributeLock distributeLock = null; + if (ClusterConfig.getInstance().isClusterEnable()) { + distributeLock = ClusterHelper.createDistributeLock(ClusterPathUtil.getConfChangeLockPath(), SystemConfig.getInstance().getInstanceName()); + if (!distributeLock.acquire()) { + service.writeErrMessage(ErrorCode.ER_YES, "Other instance is reloading, please try again later."); + return; + } + LOGGER.info("delete dble_information[{}]: added distributeLock {}", managerBaseTable.getTableName(), ClusterPathUtil.getConfChangeLockPath()); + } ManagerWritableTable managerTable = (ManagerWritableTable) managerBaseTable; int rowSize; boolean lockFlag = managerTable.getLock().tryLock(); @@ -110,29 +124,22 @@ public void handle(String stmt, ManagerService service) { } catch (Exception e) { if (e.getCause() instanceof ConfigException) { //reload fail - handleConfigException(e, service, managerTable); + service.writeErrMessage(ErrorCode.ER_YES, "Delete failure.The reason is " + e.getMessage()); + LOGGER.warn("Delete failure.The reason is " + e); } else { service.writeErrMessage(ErrorCode.ER_YES, "unknown error:" + e.getMessage()); LOGGER.warn("unknown error:", e); } return; } finally { - managerTable.deleteBackupFile(); managerTable.getLock().unlock(); + if (distributeLock != null) { + distributeLock.release(); + } } OkPacket ok = new OkPacket(); ok.setPacketId(1); ok.setAffectedRows(rowSize); ok.write(service.getConnection()); } - - private void handleConfigException(Exception e, ManagerService service, ManagerWritableTable managerTable) { - try { - managerTable.rollbackXmlFile(); - } catch (IOException ioException) { - service.writeErrMessage(ErrorCode.ER_YES, "unknown error:" + e.getMessage()); - return; - } - service.writeErrMessage(ErrorCode.ER_YES, "Delete failure.The reason is " + e.getMessage()); - } } diff --git a/src/main/java/com/actiontech/dble/services/manager/handler/InsertHandler.java b/src/main/java/com/actiontech/dble/services/manager/handler/InsertHandler.java index fe5ca9a56c..6647e3313b 100644 --- a/src/main/java/com/actiontech/dble/services/manager/handler/InsertHandler.java +++ b/src/main/java/com/actiontech/dble/services/manager/handler/InsertHandler.java @@ -5,8 +5,13 @@ package com.actiontech.dble.services.manager.handler; +import com.actiontech.dble.cluster.ClusterHelper; +import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.DistributeLock; import com.actiontech.dble.cluster.values.ConfStatus; import com.actiontech.dble.config.ErrorCode; +import com.actiontech.dble.config.model.ClusterConfig; +import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.config.util.ConfigException; import com.actiontech.dble.meta.ColumnMeta; import com.actiontech.dble.net.mysql.OkPacket; @@ -25,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -49,6 +53,15 @@ public void handle(String stmt, ManagerService service) { if (null == managerTable) { return; } + DistributeLock distributeLock = null; + if (ClusterConfig.getInstance().isClusterEnable()) { + distributeLock = ClusterHelper.createDistributeLock(ClusterPathUtil.getConfChangeLockPath(), SystemConfig.getInstance().getInstanceName()); + if (!distributeLock.acquire()) { + service.writeErrMessage(ErrorCode.ER_YES, "Other instance is reloading, please try again later."); + return; + } + LOGGER.info("insert dble_information[{}]: added distributeLock {}", managerTable.getTableName(), ClusterPathUtil.getConfChangeLockPath()); + } List columns = getColumn(insert, managerTable, service); if (null == columns) { return; @@ -67,6 +80,7 @@ public void handle(String stmt, ManagerService service) { if (rowSize != 0) { ReloadConfig.execute(service, 0, false, new ConfStatus(ConfStatus.Status.MANAGER_INSERT, managerTable.getTableName())); } + managerTable.afterExecute(); } catch (SQLException e) { service.writeErrMessage(StringUtil.isEmpty(e.getSQLState()) ? "HY000" : e.getSQLState(), e.getMessage(), e.getErrorCode()); return; @@ -75,16 +89,18 @@ public void handle(String stmt, ManagerService service) { return; } catch (Exception e) { if (e.getCause() instanceof ConfigException) { - //reload fail - handleConfigException(e, service, managerTable); + service.writeErrMessage(ErrorCode.ER_YES, "Insert failure.The reason is " + e.getMessage()); + LOGGER.warn("Insert failure.The reason is ", e); } else { service.writeErrMessage(ErrorCode.ER_YES, "unknown error:" + e.getMessage()); LOGGER.warn("unknown error:", e); } return; } finally { - managerTable.deleteBackupFile(); managerTable.getLock().unlock(); + if (distributeLock != null) { + distributeLock.release(); + } } writeOkPacket(1, rowSize, managerTable.getMsg(), service); } @@ -175,14 +191,4 @@ private ManagerWritableTable getWritableTable(MySqlInsertStatement insert, Manag } return (ManagerWritableTable) managerBaseTable; } - - private void handleConfigException(Exception e, ManagerService service, ManagerWritableTable managerTable) { - try { - managerTable.rollbackXmlFile(); - } catch (IOException ioException) { - service.writeErrMessage(ErrorCode.ER_YES, "unknown error:" + e.getMessage()); - return; - } - service.writeErrMessage(ErrorCode.ER_YES, "Insert failure.The reason is " + e.getMessage()); - } } diff --git a/src/main/java/com/actiontech/dble/services/manager/handler/UpdateHandler.java b/src/main/java/com/actiontech/dble/services/manager/handler/UpdateHandler.java index 3b9c6d0a43..2ac633021e 100644 --- a/src/main/java/com/actiontech/dble/services/manager/handler/UpdateHandler.java +++ b/src/main/java/com/actiontech/dble/services/manager/handler/UpdateHandler.java @@ -5,8 +5,13 @@ package com.actiontech.dble.services.manager.handler; +import com.actiontech.dble.cluster.ClusterHelper; +import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.DistributeLock; import com.actiontech.dble.cluster.values.ConfStatus; import com.actiontech.dble.config.ErrorCode; +import com.actiontech.dble.config.model.ClusterConfig; +import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.config.util.ConfigException; import com.actiontech.dble.net.mysql.OkPacket; import com.actiontech.dble.net.mysql.RowDataPacket; @@ -30,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.SQLException; import java.util.*; @@ -46,59 +50,26 @@ public void handle(String stmt, ManagerService service) { service.writeErrMessage("42000", "You have an error in your SQL syntax", ErrorCode.ER_PARSE_ERROR); return; } - if (update.getLimit() != null || update.isIgnore() || update.isLowPriority() || update.getOrderBy() != null) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update with syntax :[LOW_PRIORITY] [IGNORE] ... [ORDER BY ...] [LIMIT row_count]"); - return; - } - if (update.getWhere() == null) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update without WHERE"); - return; - } - SQLTableSource tableSource = update.getTableSource(); - if (tableSource instanceof SQLJoinTableSource) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update Multiple-Table "); + ManagerWritableTable managerTable = getWritableTable(update, service); + if (null == managerTable) { return; } - SQLExprTableSource singleTableSource = (SQLExprTableSource) tableSource; - if (singleTableSource.getAlias() != null) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update with alias"); - return; - } - if (singleTableSource.getPartitionSize() != 0) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update with [PARTITION (partition_name [, partition_name] ...)]"); - return; - } - ServerSchemaStatVisitor visitor = new ServerSchemaStatVisitor(); - update.accept(visitor); - if (visitor.getNotSupportMsg() != null) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, visitor.getNotSupportMsg()); - return; - } else if (visitor.getFirstClassSubQueryList().size() > 0) { - service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support sub-query"); - return; - } - SchemaUtil.SchemaInfo schemaInfo; - try { - schemaInfo = SchemaUtil.getSchemaInfo(service.getUser(), service.getSchema(), singleTableSource); - } catch (SQLException e) { - service.writeErrMessage(e.getSQLState(), e.getMessage(), e.getErrorCode()); - return; - } - ManagerBaseTable managerBaseTable = ManagerSchemaInfo.getInstance().getTables().get(schemaInfo.getTable()); - if (!managerBaseTable.isWritable()) { - service.writeErrMessage("42000", "Access denied for table '" + managerBaseTable.getTableName() + "'", ErrorCode.ER_ACCESS_DENIED_ERROR); - return; + DistributeLock distributeLock = null; + if (ClusterConfig.getInstance().isClusterEnable()) { + distributeLock = ClusterHelper.createDistributeLock(ClusterPathUtil.getConfChangeLockPath(), SystemConfig.getInstance().getInstanceName()); + if (!distributeLock.acquire()) { + service.writeErrMessage(ErrorCode.ER_YES, "Other instance is reloading, please try again later."); + return; + } + LOGGER.info("update dble_information[{}]: added distributeLock {}", managerTable.getTableName(), ClusterPathUtil.getConfChangeLockPath()); } - ManagerWritableTable managerTable = (ManagerWritableTable) managerBaseTable; - LinkedHashMap values; try { - values = getUpdateValues(schemaInfo, managerTable, update.getItems()); + values = getUpdateValues(managerTable, update.getItems()); } catch (SQLException e) { service.writeErrMessage(StringUtil.isEmpty(e.getSQLState()) ? "HY000" : e.getSQLState(), e.getMessage(), e.getErrorCode()); return; } - int rowSize; boolean lockFlag = managerTable.getLock().tryLock(); if (!lockFlag) { @@ -118,15 +89,18 @@ public void handle(String stmt, ManagerService service) { } catch (Exception e) { if (e.getCause() instanceof ConfigException) { //reload fail - handleConfigException(e, service, managerTable); + service.writeErrMessage(ErrorCode.ER_YES, "Update failure.The reason is " + e.getMessage()); + LOGGER.warn("Update failure.The reason is ", e); } else { service.writeErrMessage(ErrorCode.ER_YES, "unknown error:" + e.getMessage()); LOGGER.warn("unknown error:", e); } return; } finally { - managerTable.deleteBackupFile(); managerTable.getLock().unlock(); + if (distributeLock != null) { + distributeLock.release(); + } } OkPacket ok = new OkPacket(); ok.setPacketId(1); @@ -134,6 +108,54 @@ public void handle(String stmt, ManagerService service) { ok.write(service.getConnection()); } + + public ManagerWritableTable getWritableTable(MySqlUpdateStatement update, ManagerService service) { + if (update.getLimit() != null || update.isIgnore() || update.isLowPriority() || update.getOrderBy() != null) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update with syntax :[LOW_PRIORITY] [IGNORE] ... [ORDER BY ...] [LIMIT row_count]"); + return null; + } + if (update.getWhere() == null) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update without WHERE"); + return null; + } + SQLTableSource tableSource = update.getTableSource(); + if (tableSource instanceof SQLJoinTableSource) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update Multiple-Table "); + return null; + } + SQLExprTableSource singleTableSource = (SQLExprTableSource) tableSource; + if (singleTableSource.getAlias() != null) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update with alias"); + return null; + } + if (singleTableSource.getPartitionSize() != 0) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support update with [PARTITION (partition_name [, partition_name] ...)]"); + return null; + } + ServerSchemaStatVisitor visitor = new ServerSchemaStatVisitor(); + update.accept(visitor); + if (visitor.getNotSupportMsg() != null) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, visitor.getNotSupportMsg()); + return null; + } else if (visitor.getFirstClassSubQueryList().size() > 0) { + service.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "update syntax error, not support sub-query"); + return null; + } + SchemaUtil.SchemaInfo schemaInfo; + try { + schemaInfo = SchemaUtil.getSchemaInfo(service.getUser(), service.getSchema(), singleTableSource); + } catch (SQLException e) { + service.writeErrMessage(e.getSQLState(), e.getMessage(), e.getErrorCode()); + return null; + } + ManagerBaseTable managerBaseTable = ManagerSchemaInfo.getInstance().getTables().get(schemaInfo.getTable()); + if (!managerBaseTable.isWritable()) { + service.writeErrMessage("42000", "Access denied for table '" + managerBaseTable.getTableName() + "'", ErrorCode.ER_ACCESS_DENIED_ERROR); + return null; + } + return (ManagerWritableTable) managerBaseTable; + } + private int updateRows(ManagerService service, ManagerWritableTable managerTable, Set> affectPks, LinkedHashMap values) throws Exception { int rowSize = 0; if (!affectPks.isEmpty()) { @@ -145,11 +167,10 @@ private int updateRows(ManagerService service, ManagerWritableTable managerTable return rowSize; } - private LinkedHashMap getUpdateValues(SchemaUtil.SchemaInfo schemaInfo, - ManagerWritableTable managerTable, List updateItems) throws SQLException { + private LinkedHashMap getUpdateValues(ManagerWritableTable managerTable, List updateItems) throws SQLException { LinkedHashMap values = new LinkedHashMap<>(updateItems.size()); for (SQLUpdateSetItem item : updateItems) { - String columnName = getColumnName(item.getColumn().toString().toLowerCase(), schemaInfo.getTable()); + String columnName = getColumnName(item.getColumn().toString().toLowerCase(), managerTable.getTableName()); if (managerTable.getColumnType(columnName) == null) { throw new SQLException("Unknown column '" + columnName + "' in 'field list'", "42S22", ErrorCode.ER_BAD_FIELD_ERROR); } @@ -193,14 +214,4 @@ private String getColumnName(String columnName, String expectTableName) throws S columnName = StringUtil.removeBackQuote(columnName); return columnName; } - - private void handleConfigException(Exception e, ManagerService service, ManagerWritableTable managerTable) { - try { - managerTable.rollbackXmlFile(); - } catch (IOException ioException) { - service.writeErrMessage(ErrorCode.ER_YES, "unknown error:" + e.getMessage()); - return; - } - service.writeErrMessage(ErrorCode.ER_YES, "Update failure.The reason is " + e.getMessage()); - } } diff --git a/src/main/java/com/actiontech/dble/services/manager/information/ManagerBaseTable.java b/src/main/java/com/actiontech/dble/services/manager/information/ManagerBaseTable.java index 5d6beb0ddc..73518e34a4 100644 --- a/src/main/java/com/actiontech/dble/services/manager/information/ManagerBaseTable.java +++ b/src/main/java/com/actiontech/dble/services/manager/information/ManagerBaseTable.java @@ -17,6 +17,7 @@ public abstract class ManagerBaseTable { protected boolean isWritable = false; protected final LinkedHashMap columns; protected final LinkedHashMap columnsType; + private String msg; protected ManagerBaseTable(String tableName, int filedSize) { this.tableName = tableName; @@ -67,4 +68,12 @@ public Set getColumnNames() { public Integer getColumnType(String columnName) { return columnsType.get(columnName); } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } } diff --git a/src/main/java/com/actiontech/dble/services/manager/information/ManagerSchemaInfo.java b/src/main/java/com/actiontech/dble/services/manager/information/ManagerSchemaInfo.java index fdb6ae84b8..ebd41592c0 100644 --- a/src/main/java/com/actiontech/dble/services/manager/information/ManagerSchemaInfo.java +++ b/src/main/java/com/actiontech/dble/services/manager/information/ManagerSchemaInfo.java @@ -47,6 +47,7 @@ private ManagerSchemaInfo() { registerTable(new ProcessList()); registerTable(new SessionVariables()); registerTable(new BackendVariables()); + registerTable(new DbleConfig()); } diff --git a/src/main/java/com/actiontech/dble/services/manager/information/ManagerWritableTable.java b/src/main/java/com/actiontech/dble/services/manager/information/ManagerWritableTable.java index 79d25bd6c5..83e6d20332 100644 --- a/src/main/java/com/actiontech/dble/services/manager/information/ManagerWritableTable.java +++ b/src/main/java/com/actiontech/dble/services/manager/information/ManagerWritableTable.java @@ -5,7 +5,6 @@ package com.actiontech.dble.services.manager.information; -import com.actiontech.dble.backend.mysql.store.fs.FileUtils; import com.actiontech.dble.config.ErrorCode; import com.actiontech.dble.meta.ColumnMeta; import com.actiontech.dble.net.mysql.RowDataPacket; @@ -15,9 +14,6 @@ import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; import com.google.common.collect.Sets; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.locks.ReentrantLock; @@ -33,7 +29,6 @@ public abstract class ManagerWritableTable extends ManagerBaseTable { private Set logicalPrimaryKeySet = Sets.newHashSet(); private String xmlFilePath; - private String msg; protected ManagerWritableTable(String tableName, int filedSize) { super(tableName, filedSize); @@ -73,14 +68,6 @@ public ReentrantLock getLock() { return lock; } - public String getMsg() { - return msg; - } - - public void setMsg(String msg) { - this.msg = msg; - } - public LinkedHashSet getNotNullColumns() { return notNullColumns; } @@ -178,22 +165,6 @@ public List getRow(LinkedHashSet realSelects, String charse } } - public void rollbackXmlFile() throws IOException { - //rollback - File tempFile = new File(this.xmlFilePath + ".tmp"); - File file = new File(this.xmlFilePath); - if (tempFile.exists()) { - FileUtils.copy(tempFile, file); - } else { - throw new FileNotFoundException(tempFile.getPath()); - } - } - - public void deleteBackupFile() { - //delete - File tempFile = new File(this.xmlFilePath + ".tmp"); - if (tempFile.exists()) { - tempFile.delete(); - } + public void afterExecute() { } } diff --git a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleConfig.java b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleConfig.java new file mode 100644 index 0000000000..aacc03a183 --- /dev/null +++ b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleConfig.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2016-2021 ActionTech. + * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher. + */ +package com.actiontech.dble.services.manager.information.tables; + +import com.actiontech.dble.DbleServer; +import com.actiontech.dble.cluster.zkprocess.entity.sharding.function.Function; +import com.actiontech.dble.cluster.zkprocess.parse.JsonProcessBase; +import com.actiontech.dble.config.Fields; +import com.actiontech.dble.config.converter.ShardingConverter; +import com.actiontech.dble.meta.ColumnMeta; +import com.actiontech.dble.services.manager.information.ManagerBaseTable; +import com.actiontech.dble.util.StringUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; + +import java.lang.reflect.Type; +import java.util.LinkedHashMap; +import java.util.List; + +public class DbleConfig extends ManagerBaseTable { + + private static final String TABLE_NAME = "dble_config"; + + private static final String COLUMN_CONTENT = "content"; + + public DbleConfig() { + super(TABLE_NAME, 1); + } + + @Override + protected void initColumnAndType() { + columns.put(COLUMN_CONTENT, new ColumnMeta(COLUMN_CONTENT, "text", false)); + columnsType.put(COLUMN_CONTENT, Fields.FIELD_TYPE_STRING); + } + + @Override + protected List> getRows() { + JsonParser jsonParser = new JsonParser(); + JsonObject resultJson = new JsonObject(); + List jsonStrList = Lists.newArrayList(DbleServer.getInstance().getConfig().getDbConfig(), DbleServer.getInstance().getConfig().getShardingConfig(), + DbleServer.getInstance().getConfig().getUserConfig(), DbleServer.getInstance().getConfig().getSequenceConfig()); + for (String jsonStr : jsonStrList) { + if (StringUtil.isBlank(jsonStr)) { + continue; + } + JsonObject jsonObj = jsonParser.parse(jsonStr).getAsJsonObject(); + if (null != jsonObj && !jsonObj.isJsonNull()) { + jsonObj.entrySet().forEach(elementEntry -> { + if (elementEntry.getKey().contains("sequence")) { + elementEntry.setValue(jsonParser.parse(elementEntry.getValue().getAsString()).getAsJsonObject()); + } else if (StringUtil.equals(elementEntry.getKey(), "function")) { + JsonProcessBase base = new JsonProcessBase(); + Type parseType = new TypeToken>() { + }.getType(); + List list = base.toBeanformJson(elementEntry.getValue().toString(), parseType); + ShardingConverter.removeFileContent(list); + elementEntry.setValue(new Gson().toJsonTree(list)); + } + resultJson.add(elementEntry.getKey(), elementEntry.getValue()); + }); + } + } + LinkedHashMap rowMap = Maps.newLinkedHashMap(); + rowMap.put(COLUMN_CONTENT, resultJson.toString()); + return Lists.newArrayList(rowMap); + } +} diff --git a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbGroup.java b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbGroup.java index cf61c0b581..fa0943118f 100644 --- a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbGroup.java +++ b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbGroup.java @@ -9,9 +9,13 @@ import com.actiontech.dble.backend.datasource.PhysicalDbGroup; import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; +import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBGroup; +import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.HeartBeat; import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.DbleTempConfig; +import com.actiontech.dble.config.ErrorCode; import com.actiontech.dble.config.Fields; +import com.actiontech.dble.config.converter.DBConverter; import com.actiontech.dble.config.model.db.DbGroupConfig; import com.actiontech.dble.config.util.ConfigException; import com.actiontech.dble.meta.ColumnMeta; @@ -92,7 +96,7 @@ protected List> getRows() { } @Override - public int insertRows(List> rows) throws SQLException { + public int insertRows(List> rows) { //check rule rows.forEach(this::checkRule); tempRowList.addAll(rows); @@ -121,17 +125,31 @@ public int updateRows(Set> affectPks, LinkedHashMa if (dbGroupRows.isEmpty()) { return affectPks.size(); } - XmlProcessBase xmlProcess = new XmlProcessBase(); - DbleDbInstance dbleDbInstance = (DbleDbInstance) ManagerSchemaInfo.getInstance().getTables().get(DbleDbInstance.TABLE_NAME); - DbGroups dbs = dbleDbInstance.transformRow(xmlProcess, dbGroupRows, null); - dbs.encryptPassword(); - xmlProcess.writeObjToXml(dbs, getXmlFilePath(), "db"); + List dbGroupList = affectPks.stream().map(this::transformRowToDBGroup).collect(Collectors.toList()); + DBConverter dbConverter = new DBConverter(); + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + DbGroups dbGroups = dbConverter.dbJsonToBean(dbConfig); + + for (DBGroup dbGroup : dbGroupList) { + Optional dbGroupOp = dbGroups.getDbGroup().stream().filter(sourceDbGroup -> StringUtil.equals(sourceDbGroup.getName(), dbGroup.getName())).findFirst(); + if (!dbGroupOp.isPresent()) { + String msg = String.format("this row[%s] does not exist.", dbGroup.getName()); + throw new SQLException(msg, "42S22", ErrorCode.ER_NO_REFERENCED_ROW_2); + } + dbGroup.setDbInstance(dbGroupOp.get().getDbInstance()); + dbGroups.getDbGroup().removeIf(sourceDbGroup -> StringUtil.equals(sourceDbGroup.getName(), dbGroup.getName())); + dbGroups.getDbGroup().add(dbGroup); + } + + dbConfig = DBConverter.dbBeanToJson(dbGroups); + DbleTempConfig.getInstance().setDbConfig(dbConfig); return affectPks.size(); } + @Override - public int deleteRows(Set> affectPks) throws SQLException { + public int deleteRows(Set> affectPks) { //check checkDeleteRule(affectPks); //temp @@ -146,18 +164,55 @@ public int deleteRows(Set> affectPks) throws SQLEx return affectPks.size(); } - XmlProcessBase xmlProcess = new XmlProcessBase(); - DbleDbInstance dbleDbInstance = (DbleDbInstance) ManagerSchemaInfo.getInstance().getTables().get(DbleDbInstance.TABLE_NAME); - DbGroups dbs = dbleDbInstance.transformRow(xmlProcess, null, null); + DBConverter dbConverter = new DBConverter(); + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + DbGroups dbGroups = dbConverter.dbJsonToBean(dbConfig); for (LinkedHashMap affectPk : dbGroupRows) { - dbs.getDbGroup().removeIf(dbGroup -> StringUtil.equals(dbGroup.getName(), affectPk.get(COLUMN_NAME))); + dbGroups.getDbGroup().removeIf(dbGroup -> StringUtil.equals(dbGroup.getName(), affectPk.get(COLUMN_NAME))); } - dbs.encryptPassword(); - xmlProcess.writeObjToXml(dbs, getXmlFilePath(), "db"); + String dbGroupJson = DBConverter.dbBeanToJson(dbGroups); + DbleTempConfig.getInstance().setDbConfig(dbGroupJson); return affectPks.size(); } + + public DBGroup transformRowToDBGroup(LinkedHashMap values) { + DBGroup dbGroup = new DBGroup(); + HeartBeat heartbeat = new HeartBeat(); + dbGroup.setHeartbeat(heartbeat); + for (Map.Entry latestVal : values.entrySet()) { + String key = latestVal.getKey(); + String value = latestVal.getValue(); + switch (key) { + case COLUMN_NAME: + dbGroup.setName(value); + break; + case COLUMN_HEARTBEAT_STMT: + heartbeat.setValue(value); + break; + case COLUMN_HEARTBEAT_TIMEOUT: + heartbeat.setTimeout(Integer.parseInt(value)); + break; + case COLUMN_HEARTBEAT_RETRY: + heartbeat.setErrorRetryCount(Integer.parseInt(value)); + break; + case COLUMN_RW_SPLIT_MODE: + dbGroup.setRwSplitMode(Integer.parseInt(value)); + break; + case COLUMN_DELAY_THRESHOLD: + dbGroup.setDelayThreshold(Integer.parseInt(value)); + break; + case COLUMN_DISABLE_HA: + dbGroup.setDisableHA(value); + break; + default: + break; + } + } + return dbGroup; + } + private void checkDeleteRule(Set> affectPks) { for (LinkedHashMap affectPk : affectPks) { //check user-group @@ -187,30 +242,18 @@ public List> getTempRowList() { private void checkRule(LinkedHashMap row) { if (null != row && !row.isEmpty()) { - String delayThresholdStr = row.get(COLUMN_DELAY_THRESHOLD); - int delayThreshold = StringUtil.isEmpty(delayThresholdStr) ? 0 : Integer.parseInt(delayThresholdStr); - String disableHaStr = row.get(COLUMN_DISABLE_HA); - if (!StringUtil.isEmpty(disableHaStr) && !StringUtil.equalsIgnoreCase(disableHaStr, Boolean.FALSE.toString()) && - !StringUtil.equalsIgnoreCase(disableHaStr, Boolean.TRUE.toString())) { - throw new ConfigException("Column 'disable_ha' values only support 'false' or 'true'."); + if (row.containsKey(COLUMN_DISABLE_HA) && !StringUtil.isEmpty(row.get(COLUMN_DISABLE_HA))) { + String disableHaStr = row.get(COLUMN_DISABLE_HA); + if (!StringUtil.equalsIgnoreCase(disableHaStr, Boolean.FALSE.toString()) && + !StringUtil.equalsIgnoreCase(disableHaStr, Boolean.TRUE.toString())) { + throw new ConfigException("Column 'disable_ha' values only support 'false' or 'true'."); + } } - boolean disableHa = !StringUtil.isEmpty(disableHaStr) && Boolean.parseBoolean(disableHaStr); - String rwSplitModeStr = row.get(COLUMN_RW_SPLIT_MODE); - int rwSplitMode = StringUtil.isEmpty(rwSplitModeStr) ? 0 : Integer.parseInt(rwSplitModeStr); - String heartbeatTimeoutStr = row.get(COLUMN_HEARTBEAT_TIMEOUT); - int heartbeatTimeout = StringUtil.isEmpty(heartbeatTimeoutStr) ? 0 : Integer.parseInt(heartbeatTimeoutStr); - String heartbeatRetryStr = row.get(COLUMN_HEARTBEAT_RETRY); - int heartbeatRetry = StringUtil.isEmpty(heartbeatRetryStr) ? 0 : Integer.parseInt(heartbeatRetryStr); - DbGroupConfig dbGroupConfig = new DbGroupConfig(row.get(COLUMN_NAME), null, null, delayThreshold, disableHa); - dbGroupConfig.setRwSplitMode(rwSplitMode); - dbGroupConfig.setHeartbeatSQL(row.get(COLUMN_HEARTBEAT_STMT)); - dbGroupConfig.setHeartbeatTimeout(heartbeatTimeout * 1000); - dbGroupConfig.setErrorRetryCount(heartbeatRetry); - - LinkedHashMap map = initMap(dbGroupConfig); - for (Map.Entry entry : map.entrySet()) { - if (row.containsKey(entry.getKey())) { - row.put(entry.getKey(), entry.getValue()); + if (row.containsKey(COLUMN_RW_SPLIT_MODE) && !StringUtil.isEmpty(row.get(COLUMN_RW_SPLIT_MODE))) { + String rwSplitModeStr = row.get(COLUMN_RW_SPLIT_MODE); + int rwSplitMode = Integer.parseInt(rwSplitModeStr); + if (rwSplitMode > 2 || rwSplitMode < 0) { + throw new ConfigException("rwSplitMode should be between 0 and 2!"); } } } diff --git a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbInstance.java b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbInstance.java index 2765495543..b8a03efda0 100644 --- a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbInstance.java +++ b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleDbInstance.java @@ -7,24 +7,20 @@ import com.actiontech.dble.DbleServer; import com.actiontech.dble.backend.datasource.PhysicalDbGroup; -import com.actiontech.dble.backend.datasource.PhysicalDbInstance; import com.actiontech.dble.backend.heartbeat.MySQLHeartbeat; -import com.actiontech.dble.backend.mysql.nio.MySQLInstance; import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; import com.actiontech.dble.cluster.zkprocess.entity.Property; import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBGroup; import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.DBInstance; -import com.actiontech.dble.cluster.zkprocess.entity.dbGroups.HeartBeat; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.DbleTempConfig; import com.actiontech.dble.config.ErrorCode; import com.actiontech.dble.config.Fields; -import com.actiontech.dble.config.model.db.DbGroupConfig; +import com.actiontech.dble.config.converter.DBConverter; import com.actiontech.dble.config.model.db.DbInstanceConfig; import com.actiontech.dble.config.model.db.PoolConfig; import com.actiontech.dble.config.util.ConfigException; -import com.actiontech.dble.config.util.ParameterMapping; import com.actiontech.dble.meta.ColumnMeta; import com.actiontech.dble.services.manager.information.ManagerSchemaInfo; import com.actiontech.dble.services.manager.information.ManagerWritableTable; @@ -32,19 +28,17 @@ import com.actiontech.dble.util.DecryptUtil; import com.actiontech.dble.util.ResourceUtil; import com.actiontech.dble.util.StringUtil; +import com.google.common.base.CaseFormat; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import javax.xml.bind.JAXBException; -import javax.xml.stream.XMLStreamException; import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.sql.SQLException; import java.util.*; import java.util.stream.Collectors; + public class DbleDbInstance extends ManagerWritableTable { public static final String TABLE_NAME = "dble_db_instance"; @@ -269,105 +263,165 @@ protected List> getRows() { @Override public int insertRows(List> rows) throws SQLException { - return insertOrUpdate(rows, true); - } + List dbInstanceList = rows.stream().map(this::transformRowToDBInstance).collect(Collectors.toList()); + + DBConverter dbConverter = new DBConverter(); + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + DbGroups dbGroups = dbConverter.dbJsonToBean(dbConfig); - private int insertOrUpdate(List> rows, boolean insertFlag) throws SQLException { - decryptPassword(rows, insertFlag); - final int size = rows.size(); - XmlProcessBase xmlProcess = new XmlProcessBase(); - DbGroups dbs = transformRow(xmlProcess, null, rows); - //check logical foreign key - if (!rows.isEmpty()) { - String msg = String.format("Cannot add or update a child row: a logical foreign key '%s' constraint fails", COLUMN_DB_GROUP); - throw new SQLException(msg, "42S22", ErrorCode.ER_NO_REFERENCED_ROW_2); - } - //check primary - for (DBGroup group : dbs.getDbGroup()) { - long primaryCount = group.getDbInstance().stream().filter(dbInstance -> null != dbInstance.getPrimary() && dbInstance.getPrimary()).count(); - if (primaryCount != 1) { - String msg = String.format("dbGroup[%s] has one and only one primary instance", group.getName()); - throw new ConfigException(msg); - } - } - //check connection - checkInstanceConnection(dbs); - //remove temp row DbleDbGroup dbleDbGroup = (DbleDbGroup) ManagerSchemaInfo.getInstance().getTables().get(DbleDbGroup.TABLE_NAME); - for (DBGroup dbGroup : dbs.getDbGroup()) { - dbleDbGroup.getTempRowList().removeIf(group -> StringUtil.equals(group.get(DbleDbGroup.COLUMN_NAME), dbGroup.getName())); + List> tempDbGroupMapList = dbleDbGroup.getTempRowList(); + List tempDbGroupList = tempDbGroupMapList.stream().map(dbleDbGroup::transformRowToDBGroup).collect(Collectors.toList()); + + for (DBInstance dbInstance : dbInstanceList) { + Optional dbGroupOp = dbGroups.getDbGroup().stream().filter(dbGroup -> StringUtil.equals(dbGroup.getName(), dbInstance.getDbGroup())).findFirst(); + if (!dbGroupOp.isPresent()) { + dbGroupOp = tempDbGroupList.stream().filter(dbGroup -> StringUtil.equals(dbGroup.getName(), dbInstance.getDbGroup())).findFirst(); + if (!dbGroupOp.isPresent()) { + String msg = String.format("Cannot add or update a child row: a logical foreign key '%s':%s constraint fails", COLUMN_DB_GROUP, dbInstance.getDbGroup()); + throw new SQLException(msg, "42S22", ErrorCode.ER_NO_REFERENCED_ROW_2); + } + dbGroups.addDbGroup(dbGroupOp.get()); + } + dbGroupOp.get().addDbInstance(dbInstance); } - dbs.encryptPassword(); - //write to configuration - xmlProcess.writeObjToXml(dbs, getXmlFilePath(), "db"); - return size; - } + dbConfig = DBConverter.dbBeanToJson(dbGroups); + DbleTempConfig.getInstance().setDbConfig(dbConfig); + return rows.size(); + } @Override public int updateRows(Set> affectPks, LinkedHashMap values) throws SQLException { affectPks.forEach(affectPk -> { - if (values.containsKey(COLUMN_PASSWORD_ENCRYPT) && Boolean.FALSE.toString().equalsIgnoreCase(values.get(COLUMN_ENCRYPT_CONFIGURED))) { - boolean containsName = values.containsKey(COLUMN_NAME); - boolean containsUser = values.containsKey(COLUMN_USER); - values.put(COLUMN_PASSWORD_ENCRYPT, getPasswordEncrypt(containsName ? values.get(COLUMN_NAME) : affectPk.get(COLUMN_NAME), containsUser ? values.get(COLUMN_USER) : affectPk.get(COLUMN_USER), values.get(COLUMN_PASSWORD_ENCRYPT))); + if (Boolean.FALSE.toString().equalsIgnoreCase(affectPk.get(COLUMN_ENCRYPT_CONFIGURED))) { + String password = DecryptUtil.dbHostDecrypt(true, affectPk.get(COLUMN_NAME), affectPk.get(COLUMN_USER), affectPk.get(COLUMN_PASSWORD_ENCRYPT)); + affectPk.put(COLUMN_PASSWORD_ENCRYPT, password); } affectPk.putAll(values); }); - return insertOrUpdate(Lists.newArrayList(affectPks), false); + List dbInstanceList = affectPks.stream().map(this::transformRowToDBInstance).collect(Collectors.toList()); + + DBConverter dbConverter = new DBConverter(); + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + DbGroups dbGroups = dbConverter.dbJsonToBean(dbConfig); + + for (DBInstance dbInstance : dbInstanceList) { + Optional dbGroupOp = dbGroups.getDbGroup().stream().filter(dbGroup -> StringUtil.equals(dbGroup.getName(), dbInstance.getDbGroup())).findFirst(); + if (!dbGroupOp.isPresent()) { + String msg = String.format("Cannot add or update a child row: a logical foreign key '%s':%s constraint fails", COLUMN_DB_GROUP, dbInstance.getDbGroup()); + throw new SQLException(msg, "42S22", ErrorCode.ER_NO_REFERENCED_ROW_2); + } + dbGroupOp.get().getDbInstance().removeIf(sourceDbInstance -> StringUtil.equals(sourceDbInstance.getName(), dbInstance.getName())); + dbGroupOp.get().addDbInstance(dbInstance); + } + dbConfig = DBConverter.dbBeanToJson(dbGroups); + DbleTempConfig.getInstance().setDbConfig(dbConfig); + return affectPks.size(); } @Override public int deleteRows(Set> affectPks) throws SQLException { - XmlProcessBase xmlProcess = new XmlProcessBase(); - DbGroups dbGroups = transformRow(xmlProcess, null, null); - for (LinkedHashMap affectPk : affectPks) { - for (DBGroup dbGroup : dbGroups.getDbGroup()) { - dbGroup.getDbInstance().removeIf(dbInstance -> StringUtil.equals(affectPk.get(COLUMN_DB_GROUP), dbGroup.getName()) && StringUtil.equals(affectPk.get(COLUMN_NAME), dbInstance.getName())); - } - } - for (DBGroup dbGroup : dbGroups.getDbGroup()) { - boolean existPrimary = dbGroup.getDbInstance().stream().anyMatch(dbInstance -> null != dbInstance.getPrimary() && dbInstance.getPrimary()); - if (!existPrimary && !dbGroup.getDbInstance().isEmpty()) { - throw new SQLException("Table dble_db_group[" + dbGroup.getName() + "] needs to retain a primary dbInstance", "42S22", ErrorCode.ER_YES); + List dbInstanceList = affectPks.stream().map(this::transformRowToDBInstance).collect(Collectors.toList()); + + DBConverter dbConverter = new DBConverter(); + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + DbGroups dbGroups = dbConverter.dbJsonToBean(dbConfig); + + for (DBInstance dbInstance : dbInstanceList) { + Optional dbGroupOp = dbGroups.getDbGroup().stream().filter(dbGroup -> StringUtil.equals(dbGroup.getName(), dbInstance.getDbGroup())).findFirst(); + if (!dbGroupOp.isPresent()) { + String msg = String.format("Cannot add or update a child row: a logical foreign key '%s':%s constraint fails", COLUMN_DB_GROUP, dbInstance.getDbGroup()); + throw new SQLException(msg, "42S22", ErrorCode.ER_NO_REFERENCED_ROW_2); } + dbGroupOp.get().getDbInstance().removeIf(sourceDbInstance -> StringUtil.equals(sourceDbInstance.getName(), dbInstance.getName())); } - Set removeDBGroupSet = dbGroups.getDbGroup().stream().filter(dbGroup -> dbGroup.getDbInstance().isEmpty()).collect(Collectors.toSet()); - //check remove empty instance - checkDeleteRule(removeDBGroupSet); //remove empty instance dbGroups.getDbGroup().removeIf(dbGroup -> dbGroup.getDbInstance().isEmpty()); - dbGroups.encryptPassword(); - //write to configuration - xmlProcess.writeObjToXml(dbGroups, getXmlFilePath(), "db"); + dbConfig = DBConverter.dbBeanToJson(dbGroups); + DbleTempConfig.getInstance().setDbConfig(dbConfig); return affectPks.size(); } - private void checkDeleteRule(Set removeDBGroupSet) { - for (DBGroup dbGroup : removeDBGroupSet) { - //check user-group - DbleRwSplitEntry dbleRwSplitEntry = (DbleRwSplitEntry) ManagerSchemaInfo.getInstance().getTables().get(DbleRwSplitEntry.TABLE_NAME); - boolean existUser = dbleRwSplitEntry.getRows().stream().anyMatch(entry -> entry.get(DbleRwSplitEntry.COLUMN_DB_GROUP).equals(dbGroup.getName())); - if (existUser) { - throw new ConfigException("Cannot delete or update a parent row: a foreign key constraint fails `dble_db_user`(`db_group`) REFERENCES `dble_db_group`(`name`)"); - } - //check sharding_node-group - DbleShardingNode dbleShardingNode = (DbleShardingNode) ManagerSchemaInfo.getInstance().getTables().get(DbleShardingNode.TABLE_NAME); - boolean existShardingNode = dbleShardingNode.getRows().stream().anyMatch(entry -> entry.get(DbleShardingNode.COLUMN_DB_GROUP).equals(dbGroup.getName())); - if (existShardingNode) { - throw new ConfigException("Cannot delete or update a parent row: a foreign key constraint fails `dble_sharding_node`(`db_group`) REFERENCES `dble_db_group`(`name`)"); - } + @Override + public void afterExecute() { + //remove temp dbGroup + DBConverter dbConverter = new DBConverter(); + String dbConfig = DbleServer.getInstance().getConfig().getDbConfig(); + DbGroups dbGroups = dbConverter.dbJsonToBean(dbConfig); + DbleDbGroup dbleDbGroup = (DbleDbGroup) ManagerSchemaInfo.getInstance().getTables().get(DbleDbGroup.TABLE_NAME); + for (DBGroup dbGroup : dbGroups.getDbGroup()) { + dbleDbGroup.getTempRowList().removeIf(group -> StringUtil.equals(group.get(DbleDbGroup.COLUMN_NAME), dbGroup.getName())); } } - private void decryptPassword(List> rows, boolean insertFlag) { - for (LinkedHashMap row : rows) { - checkBooleanVal(row); - if ((insertFlag && Boolean.parseBoolean(row.get(COLUMN_ENCRYPT_CONFIGURED))) || !insertFlag) { - row.put(COLUMN_PASSWORD_ENCRYPT, DecryptUtil.dbHostDecrypt(true, row.get(COLUMN_NAME), - row.get(COLUMN_USER), row.get(COLUMN_PASSWORD_ENCRYPT))); + private DBInstance transformRowToDBInstance(LinkedHashMap map) { + if (null == map || map.isEmpty()) { + return null; + } + checkBooleanVal(map); + DBInstance dbInstance = new DBInstance(); + StringBuilder url = new StringBuilder(); + List propertyList = Lists.newArrayList(); + for (Map.Entry entry : map.entrySet()) { + switch (entry.getKey()) { + case COLUMN_NAME: + dbInstance.setName(entry.getValue()); + break; + case COLUMN_DB_GROUP: + dbInstance.setDbGroup(entry.getValue()); + break; + case COLUMN_ADDR: + case COLUMN_PORT: + url.append(entry.getValue()).append(":"); + break; + case COLUMN_USER: + dbInstance.setUser(entry.getValue()); + break; + case COLUMN_PASSWORD_ENCRYPT: + dbInstance.setPassword(entry.getValue()); + break; + case COLUMN_ENCRYPT_CONFIGURED: + dbInstance.setUsingDecrypt(entry.getValue()); + break; + case COLUMN_PRIMARY: + dbInstance.setPrimary(!StringUtil.isEmpty(entry.getValue()) && Boolean.parseBoolean(entry.getValue())); + break; + case COLUMN_DISABLED: + dbInstance.setDisabled(entry.getValue()); + break; + case COLUMN_MIN_CONN_COUNT: + dbInstance.setMinCon(Integer.parseInt(entry.getValue())); + break; + case COLUMN_MAX_CONN_COUNT: + dbInstance.setMaxCon(Integer.parseInt(entry.getValue())); + break; + case COLUMN_READ_WEIGHT: + dbInstance.setReadWeight(entry.getValue()); + break; + case COLUMN_ID: + dbInstance.setId(entry.getValue()); + break; + case COLUMN_TEST_ON_CREATE: + case COLUMN_TEST_ON_BORROW: + case COLUMN_TEST_ON_RETURN: + case COLUMN_TEST_WHILE_IDLE: + case COLUMN_CONNECTION_TIMEOUT: + case COLUMN_CONNECTION_HEARTBEAT_TIMEOUT: + case COLUMN_TIME_BETWEEN_EVICTION_RUNS_MILLIS: + case COLUMN_IDLE_TIMEOUT: + case COLUMN_HEARTBEAT_PERIOD_MILLIS: + case COLUMN_EVICTOR_SHUTDOWN_TIMEOUT_MILLIS: + String key = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, entry.getKey()); + propertyList.add(new Property(entry.getValue(), key)); + break; + default: + break; } } + dbInstance.setUrl(url.substring(0, url.length() - 1)); + dbInstance.setProperty(propertyList); + return dbInstance; } private void checkBooleanVal(LinkedHashMap row) { @@ -383,181 +437,7 @@ private void checkBooleanVal(LinkedHashMap row) { } } - - public DbGroups transformRow(XmlProcessBase xmlProcess, List> changeDbGroupRows, List> changeDbInstanceRows) { - if (null == xmlProcess) { - return null; - } - DbGroups dbs = null; - try { - xmlProcess.addParseClass(DbGroups.class); - xmlProcess.initJaxbClass(); - dbs = (DbGroups) xmlProcess.baseParseXmlToBean(ConfigFileName.DB_XML); - } catch (JAXBException | XMLStreamException e) { - e.printStackTrace(); - } - if (null == dbs) { - throw new ConfigException("configuration is empty"); - } - for (DBGroup dbGroup : dbs.getDbGroup()) { - for (DBInstance dbInstance : dbGroup.getDbInstance()) { - String usingDecrypt = dbInstance.getUsingDecrypt(); - if (!StringUtil.isEmpty(usingDecrypt) && Boolean.parseBoolean(usingDecrypt)) { - dbInstance.setPassword(DecryptUtil.dbHostDecrypt(true, dbInstance.getName(), dbInstance.getUser(), dbInstance.getPassword())); - } - } - } - DbleDbGroup dbleDbGroup = (DbleDbGroup) ManagerSchemaInfo.getInstance().getTables().get(DbleDbGroup.TABLE_NAME); - List> dbGroupRowList = dbleDbGroup.getRows(); - for (LinkedHashMap dbGroupRow : dbGroupRowList) { - DBGroup dbGroup = initDBGroup(dbGroupRow, changeDbGroupRows, dbs); - initDBInstance(dbGroupRow, changeDbInstanceRows, dbGroup); - dbs.addDbGroup(dbGroup); - } - dbs.getDbGroup().removeIf(dbGroup -> null == dbGroup.getDbInstance() || dbGroup.getDbInstance().isEmpty()); - return dbs; - } - - private void initDBInstance(LinkedHashMap dbGroupRow, List> changeDbInstanceRows, DBGroup dbGroup) { - if (null == changeDbInstanceRows) { - return; - } - List> instanceRowList = changeDbInstanceRows.stream().filter(row -> StringUtil.equals(dbGroupRow.get(DbleDbGroup.COLUMN_NAME), row.get(COLUMN_DB_GROUP))).collect(Collectors.toList()); - if (!instanceRowList.isEmpty()) { - instanceRowList.forEach(instanceRowMap -> { - List propertyList = Lists.newArrayList(); - String testOnCreate = instanceRowMap.get(COLUMN_TEST_ON_CREATE); - if (!StringUtil.isEmpty(testOnCreate)) { - propertyList.add(new Property(testOnCreate, "testOnCreate")); - } - String testOnBorrow = instanceRowMap.get(COLUMN_TEST_ON_BORROW); - if (!StringUtil.isEmpty(testOnBorrow)) { - propertyList.add(new Property(testOnBorrow, "testOnBorrow")); - } - String testOnReturn = instanceRowMap.get(COLUMN_TEST_ON_RETURN); - if (!StringUtil.isEmpty(testOnReturn)) { - propertyList.add(new Property(testOnReturn, "testOnReturn")); - } - String testWhileIdle = instanceRowMap.get(COLUMN_TEST_WHILE_IDLE); - if (!StringUtil.isEmpty(testWhileIdle)) { - propertyList.add(new Property(testWhileIdle, "testWhileIdle")); - } - String connectionTimeout = instanceRowMap.get(COLUMN_CONNECTION_TIMEOUT); - if (!StringUtil.isEmpty(connectionTimeout)) { - propertyList.add(new Property(connectionTimeout, "connectionTimeout")); - } - String connectionHeartbeatTimeout = instanceRowMap.get(COLUMN_CONNECTION_HEARTBEAT_TIMEOUT); - if (!StringUtil.isEmpty(connectionHeartbeatTimeout)) { - propertyList.add(new Property(connectionHeartbeatTimeout, "connectionHeartbeatTimeout")); - } - String timeBetweenEvictionRunsMillis = instanceRowMap.get(COLUMN_TIME_BETWEEN_EVICTION_RUNS_MILLIS); - if (!StringUtil.isEmpty(timeBetweenEvictionRunsMillis)) { - propertyList.add(new Property(timeBetweenEvictionRunsMillis, "timeBetweenEvictionRunsMillis")); - } - String idleTimeout = instanceRowMap.get(COLUMN_IDLE_TIMEOUT); - if (!StringUtil.isEmpty(idleTimeout)) { - propertyList.add(new Property(idleTimeout, "idleTimeout")); - } - String heartbeatPeriodMillis = instanceRowMap.get(COLUMN_HEARTBEAT_PERIOD_MILLIS); - if (!StringUtil.isEmpty(heartbeatPeriodMillis)) { - propertyList.add(new Property(heartbeatPeriodMillis, "heartbeatPeriodMillis")); - } - String evictorShutdownTimeoutMillis = instanceRowMap.get(COLUMN_EVICTOR_SHUTDOWN_TIMEOUT_MILLIS); - if (!StringUtil.isEmpty(evictorShutdownTimeoutMillis)) { - propertyList.add(new Property(evictorShutdownTimeoutMillis, "evictorShutdownTimeoutMillis")); - } - Integer maxCon = StringUtil.isEmpty(instanceRowMap.get(COLUMN_MAX_CONN_COUNT)) ? null : Integer.valueOf(instanceRowMap.get(COLUMN_MAX_CONN_COUNT)); - Integer minCon = StringUtil.isEmpty(instanceRowMap.get(COLUMN_MIN_CONN_COUNT)) ? null : Integer.valueOf(instanceRowMap.get(COLUMN_MIN_CONN_COUNT)); - Boolean primary = StringUtil.isEmpty(instanceRowMap.get(COLUMN_PRIMARY)) ? null : Boolean.valueOf(instanceRowMap.get(COLUMN_PRIMARY)); - DBInstance dbInstance = new DBInstance(instanceRowMap.get(COLUMN_NAME), instanceRowMap.get(COLUMN_ADDR) + ":" + instanceRowMap.get(COLUMN_PORT), - instanceRowMap.get(COLUMN_PASSWORD_ENCRYPT), instanceRowMap.get(COLUMN_USER), maxCon, minCon, instanceRowMap.get(COLUMN_DISABLED), - instanceRowMap.get(COLUMN_ID), instanceRowMap.get(COLUMN_READ_WEIGHT), primary, propertyList, instanceRowMap.get(COLUMN_ENCRYPT_CONFIGURED)); - if (dbGroup.getDbInstance().stream().anyMatch(instance -> StringUtil.equals(instance.getName(), dbInstance.getName()))) { - dbGroup.getDbInstance().removeIf(instance -> StringUtil.equals(instance.getName(), dbInstance.getName())); - } - dbGroup.addDbInstance(dbInstance); - changeDbInstanceRows.remove(instanceRowMap); - }); - } - } - - private DBGroup initDBGroup(LinkedHashMap dbGroupRow, List> changeDbGroupRows, DbGroups dbs) { - changeDbGroupRows = null != changeDbGroupRows ? changeDbGroupRows : Lists.newArrayList(); - LinkedHashMap finalDbGroupRow = dbGroupRow; - Optional> changeDbGroupRow = changeDbGroupRows.stream().filter(changeGroupRow -> StringUtil.equals(changeGroupRow.get(DbleDbGroup.COLUMN_NAME), finalDbGroupRow.get(COLUMN_NAME))).findFirst(); - if (changeDbGroupRow.isPresent()) { - dbGroupRow = changeDbGroupRow.get(); - } - Integer timeout = StringUtil.isEmpty(dbGroupRow.get(DbleDbGroup.COLUMN_HEARTBEAT_TIMEOUT)) ? null : Integer.valueOf(dbGroupRow.get(DbleDbGroup.COLUMN_HEARTBEAT_TIMEOUT)); - Integer errorRetryCount = StringUtil.isEmpty(dbGroupRow.get(DbleDbGroup.COLUMN_HEARTBEAT_RETRY)) ? null : Integer.valueOf(dbGroupRow.get(DbleDbGroup.COLUMN_HEARTBEAT_RETRY)); - Integer rwSplitMode = StringUtil.isEmpty(dbGroupRow.get(DbleDbGroup.COLUMN_RW_SPLIT_MODE)) ? null : Integer.valueOf(dbGroupRow.get(DbleDbGroup.COLUMN_RW_SPLIT_MODE)); - Integer delayThreshold = StringUtil.isEmpty(dbGroupRow.get(DbleDbGroup.COLUMN_DELAY_THRESHOLD)) ? null : Integer.valueOf(dbGroupRow.get(DbleDbGroup.COLUMN_DELAY_THRESHOLD)); - HeartBeat heartBeat = new HeartBeat(dbGroupRow.get(DbleDbGroup.COLUMN_HEARTBEAT_STMT), timeout, errorRetryCount); - DBGroup dbGroup = new DBGroup(rwSplitMode, dbGroupRow.get(DbleDbGroup.COLUMN_NAME), delayThreshold, dbGroupRow.get(DbleDbGroup.COLUMN_DISABLE_HA), heartBeat); - Optional first = dbs.getDbGroup().stream().filter(group -> StringUtil.equals(group.getName(), dbGroup.getName())).findFirst(); - if (first.isPresent()) { - DBGroup oldDbGroup = first.get(); - dbs.getDbGroup().removeIf(group -> StringUtil.equals(group.getName(), dbGroup.getName())); - dbGroup.addAllDbInstance(oldDbGroup.getDbInstance()); - } - return dbGroup; - } - - private void checkInstanceConnection(DbGroups dbs) { - try { - for (DBGroup dbGroup : dbs.getDbGroup()) { - List dbInstanceList = dbGroup.getDbInstance(); - DbInstanceConfig tmpDbInstanceConfig = null; - List dbInstanceConfigList = Lists.newArrayList(); - for (DBInstance dbInstance : dbInstanceList) { - String url = dbInstance.getUrl(); - int colonIndex = url.indexOf(':'); - String ip = url.substring(0, colonIndex).trim(); - int port = Integer.parseInt(url.substring(colonIndex + 1).trim()); - boolean disabled = !StringUtil.isEmpty(dbInstance.getDisabled()) && Boolean.parseBoolean(dbInstance.getDisabled()); - int readWeight = StringUtil.isEmpty(dbInstance.getReadWeight()) ? 0 : Integer.parseInt(dbInstance.getReadWeight()); - boolean usingDecrypt = !StringUtil.isEmpty(dbInstance.getUsingDecrypt()) && Boolean.parseBoolean(dbInstance.getUsingDecrypt()); - List propertyList = dbInstance.getProperty(); - PoolConfig poolConfig = null; - if (!propertyList.isEmpty()) { - Map propertyMap = propertyList.stream().collect(Collectors.toMap(Property::getName, Property::getValue)); - poolConfig = new PoolConfig(); - ParameterMapping.mapping(poolConfig, propertyMap, null); - } - String password = dbInstance.getPassword(); - boolean primary = (null == dbInstance.getPrimary() ? false : dbInstance.getPrimary()); - if (primary) { - tmpDbInstanceConfig = new DbInstanceConfig(dbInstance.getName(), ip, port, url, dbInstance.getUser(), password, readWeight, dbInstance.getId(), - disabled, true, dbInstance.getMaxCon(), dbInstance.getMinCon(), poolConfig, usingDecrypt); - } else { - dbInstanceConfigList.add(new DbInstanceConfig(dbInstance.getName(), ip, port, url, dbInstance.getUser(), password, readWeight, dbInstance.getId(), - disabled, false, dbInstance.getMaxCon(), dbInstance.getMinCon(), poolConfig, usingDecrypt)); - } - } - boolean disableHA = !StringUtil.isEmpty(dbGroup.getDisableHA()) && Boolean.parseBoolean(dbGroup.getDisableHA()); - DbInstanceConfig[] dbInstanceConfigs = dbInstanceConfigList.isEmpty() ? new DbInstanceConfig[0] : dbInstanceConfigList.toArray(new DbInstanceConfig[0]); - DbGroupConfig dbGroupConf = new DbGroupConfig(dbGroup.getName(), tmpDbInstanceConfig, dbInstanceConfigs, dbGroup.getDelayThreshold(), disableHA); - //test connection - PhysicalDbInstance writeSource = new MySQLInstance(dbGroupConf.getWriteInstanceConfig(), dbGroupConf, true); - boolean isConnected = writeSource.testConnection(); - if (!isConnected) { - throw new ConfigException("Can't connect to [" + dbGroupConf.getName() + "," + writeSource.getName() + "," + writeSource.getConfig().getUrl() + "]"); - } - for (DbInstanceConfig readInstanceConfig : dbGroupConf.getReadInstanceConfigs()) { - MySQLInstance readInstance = new MySQLInstance(readInstanceConfig, dbGroupConf, false); - isConnected = readInstance.testConnection(); - if (!isConnected) { - throw new ConfigException("Can't connect to [" + dbGroupConf.getName() + "," + readInstance.getName() + "," + readInstance.getConfig().getUrl() + "]"); - } - } - } - } catch (IllegalAccessException | InvocationTargetException | IOException e) { - throw new ConfigException(e); - } - } - - - public static String getPasswordEncrypt(String instanceName, String name, String password) { + private String getPasswordEncrypt(String instanceName, String name, String password) { try { return DecryptUtil.encrypt("1:" + instanceName + ":" + name + ":" + password); } catch (Exception e) { diff --git a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleRwSplitEntry.java b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleRwSplitEntry.java index 442b39e651..2351c7caef 100644 --- a/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleRwSplitEntry.java +++ b/src/main/java/com/actiontech/dble/services/manager/information/tables/DbleRwSplitEntry.java @@ -3,15 +3,19 @@ import com.actiontech.dble.DbleServer; import com.actiontech.dble.backend.datasource.PhysicalDbGroup; import com.actiontech.dble.cluster.ClusterPathUtil; +import com.actiontech.dble.cluster.zkprocess.entity.Users; +import com.actiontech.dble.cluster.zkprocess.entity.user.RwSplitUser; import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.DbleTempConfig; import com.actiontech.dble.config.ErrorCode; import com.actiontech.dble.config.Fields; -import com.actiontech.dble.config.loader.xml.XMLUserLoader; +import com.actiontech.dble.config.converter.UserConverter; import com.actiontech.dble.config.model.user.RwSplitUserConfig; import com.actiontech.dble.config.model.user.UserConfig; import com.actiontech.dble.config.util.ConfigException; import com.actiontech.dble.meta.ColumnMeta; import com.actiontech.dble.services.manager.information.ManagerWritableTable; +import com.actiontech.dble.util.DecryptUtil; import com.actiontech.dble.util.IPAddressUtil; import com.actiontech.dble.util.ResourceUtil; import com.actiontech.dble.util.StringUtil; @@ -114,37 +118,90 @@ protected List> getRows() { @Override public int insertRows(List> rows) throws SQLException { - for (LinkedHashMap row : rows) { - check(row); - } - //write to configuration - List> tempRowList = rows.stream().map(this::transformRow).collect(Collectors.toList()); - XMLUserLoader xmlUserLoader = new XMLUserLoader(); - xmlUserLoader.insertRwSplitUser(tempRowList, getXmlFilePath()); + List rwSplitUserList = rows.stream().map(this::transformRowToUser).collect(Collectors.toList()); + + UserConverter userConverter = new UserConverter(); + String userConfig = DbleServer.getInstance().getConfig().getUserConfig(); + Users users = userConverter.userJsonToBean(userConfig); + + checkLogicalUniqueKeyDuplicate(users, rwSplitUserList); + + users.getUser().addAll(rwSplitUserList); + userConfig = userConverter.userBeanToJson(users); + DbleTempConfig.getInstance().setUserConfig(userConfig); return rows.size(); } + @Override - public int updateRows(Set> affectPks, LinkedHashMap values) throws SQLException { - check(values); - //write to configuration - List> tempRowList = affectPks.stream().map(this::transformRow).collect(Collectors.toList()); - XMLUserLoader xmlUserLoader = new XMLUserLoader(); - xmlUserLoader.updateRwSplitUser(tempRowList, transformRow(values), getXmlFilePath()); + public int updateRows(Set> affectPks, LinkedHashMap values) { + affectPks.forEach(affectPk -> { + if (Boolean.FALSE.toString().equalsIgnoreCase(affectPk.get(COLUMN_ENCRYPT_CONFIGURED))) { + String password = DecryptUtil.decrypt(true, affectPk.get(COLUMN_USERNAME), affectPk.get(COLUMN_PASSWORD_ENCRYPT)); + affectPk.put(COLUMN_PASSWORD_ENCRYPT, password); + } + affectPk.putAll(values); + }); + List rwSplitUserList = affectPks.stream().map(this::transformRowToUser).collect(Collectors.toList()); + + UserConverter userConverter = new UserConverter(); + String userConfig = DbleServer.getInstance().getConfig().getUserConfig(); + Users users = userConverter.userJsonToBean(userConfig); + + updateList(users, rwSplitUserList, false); + userConfig = userConverter.userBeanToJson(users); + DbleTempConfig.getInstance().setUserConfig(userConfig); + return affectPks.size(); } @Override - public int deleteRows(Set> affectPks) throws SQLException { - //write to configuration - List> tempRowList = affectPks.stream().map(this::transformRow).collect(Collectors.toList()); - XMLUserLoader xmlUserLoader = new XMLUserLoader(); - xmlUserLoader.deleteRwSplitUser(tempRowList, getXmlFilePath()); + public int deleteRows(Set> affectPks) { + List rwSplitUserList = affectPks.stream().map(this::transformRowToUser).collect(Collectors.toList()); + + UserConverter userConverter = new UserConverter(); + String userConfig = DbleServer.getInstance().getConfig().getUserConfig(); + Users users = userConverter.userJsonToBean(userConfig); + + updateList(users, rwSplitUserList, true); + userConfig = userConverter.userBeanToJson(users); + DbleTempConfig.getInstance().setUserConfig(userConfig); return affectPks.size(); } - private void check(LinkedHashMap tempRowMap) throws SQLException { + private void checkLogicalUniqueKeyDuplicate(Users users, List rwSplitUserList) throws SQLException { + List sourceList = users.getUser().stream().filter(user -> user instanceof RwSplitUser).map(user -> (RwSplitUser) user).collect(Collectors.toList()); + for (RwSplitUser rwSplitUser : rwSplitUserList) { + boolean isExist = sourceList.stream().anyMatch(sourceUser -> StringUtil.equals(sourceUser.getName(), rwSplitUser.getName()) && StringUtil.equals(sourceUser.getTenant(), rwSplitUser.getTenant())); + if (isExist) { + String msg = String.format("Duplicate entry '%s-%s-%s'for logical unique '%s-%s-%s'", rwSplitUser.getName(), + StringUtil.isEmpty(rwSplitUser.getTenant()) ? null : "tenant", rwSplitUser.getTenant(), COLUMN_USERNAME, COLUMN_CONN_ATTR_KEY, COLUMN_CONN_ATTR_VALUE); + throw new SQLException(msg, "42S22", ErrorCode.ER_DUP_ENTRY); + } + } + } + + private void updateList(Users users, List rwSplitUserList, boolean isDelete) { + for (RwSplitUser rwSplitUser : rwSplitUserList) { + for (int i = 0; i < users.getUser().size(); i++) { + Object obj = users.getUser().get(i); + if (obj instanceof RwSplitUser) { + RwSplitUser sourceUser = (RwSplitUser) obj; + if (StringUtil.equals(sourceUser.getName(), rwSplitUser.getName()) && StringUtil.equals(sourceUser.getTenant(), rwSplitUser.getTenant())) { + if (!isDelete) { + users.getUser().set(i, rwSplitUser); + } else { + users.getUser().remove(i); + } + break; + } + } + } + } + } + + private void check(LinkedHashMap tempRowMap) { //check whiteIPs checkWhiteIPs(tempRowMap); //check db_group @@ -169,42 +226,49 @@ private void checkWhiteIPs(LinkedHashMap tempRowMap) { } } - private void checkDbGroup(LinkedHashMap tempRowMap) throws SQLException { + private void checkDbGroup(LinkedHashMap tempRowMap) { if (tempRowMap.containsKey(COLUMN_DB_GROUP)) { Map dbGroupMap = DbleServer.getInstance().getConfig().getDbGroups(); boolean isExist = dbGroupMap.keySet().stream().anyMatch(groupName -> StringUtil.equals(groupName, tempRowMap.get(COLUMN_DB_GROUP))); if (!isExist) { - throw new SQLException("Column 'db_group' value '" + tempRowMap.get(COLUMN_DB_GROUP) + "' does not exist or not active.", "42S22", ErrorCode.ER_ERROR_ON_WRITE); + throw new ConfigException("Column 'db_group' value '" + tempRowMap.get(COLUMN_DB_GROUP) + "' does not exist or not active."); } } } - private LinkedHashMap transformRow(LinkedHashMap map) { + private RwSplitUser transformRowToUser(LinkedHashMap map) { if (null == map || map.isEmpty()) { return null; } - LinkedHashMap xmlMap = Maps.newLinkedHashMap(); - if (null != map.get(COLUMN_USERNAME)) { - xmlMap.put("name", map.get(COLUMN_USERNAME)); - } - if (null != map.get(COLUMN_PASSWORD_ENCRYPT)) { - xmlMap.put("password", map.get(COLUMN_PASSWORD_ENCRYPT)); - } - if (null != map.get(COLUMN_WHITE_IPS)) { - xmlMap.put("whiteIPs", map.get(COLUMN_WHITE_IPS)); - } - if (null != map.get(COLUMN_MAX_CONN_COUNT)) { - xmlMap.put("maxCon", map.get(COLUMN_MAX_CONN_COUNT)); - } - if (null != map.get(COLUMN_CONN_ATTR_VALUE)) { - xmlMap.put("tenant", map.get(COLUMN_CONN_ATTR_VALUE)); - } - if (null != map.get(COLUMN_DB_GROUP)) { - xmlMap.put("dbGroup", map.get(COLUMN_DB_GROUP)); - } - if (null != map.get(COLUMN_ENCRYPT_CONFIGURED)) { - xmlMap.put("usingDecrypt", map.get(COLUMN_ENCRYPT_CONFIGURED)); + check(map); + RwSplitUser rwSplitUser = new RwSplitUser(); + for (Map.Entry entry : map.entrySet()) { + switch (entry.getKey()) { + case COLUMN_USERNAME: + rwSplitUser.setName(entry.getValue()); + break; + case COLUMN_PASSWORD_ENCRYPT: + rwSplitUser.setPassword(entry.getValue()); + break; + case COLUMN_ENCRYPT_CONFIGURED: + rwSplitUser.setUsingDecrypt(entry.getValue()); + break; + case COLUMN_CONN_ATTR_VALUE: + rwSplitUser.setTenant(entry.getValue()); + break; + case COLUMN_WHITE_IPS: + rwSplitUser.setWhiteIPs(entry.getValue()); + break; + case COLUMN_MAX_CONN_COUNT: + rwSplitUser.setMaxCon(Integer.parseInt(entry.getValue())); + break; + case COLUMN_DB_GROUP: + rwSplitUser.setDbGroup(entry.getValue()); + break; + default: + break; + } } - return xmlMap; + return rwSplitUser; } } diff --git a/src/main/java/com/actiontech/dble/services/manager/response/DryRun.java b/src/main/java/com/actiontech/dble/services/manager/response/DryRun.java index e9563d89d0..6e58077e56 100644 --- a/src/main/java/com/actiontech/dble/services/manager/response/DryRun.java +++ b/src/main/java/com/actiontech/dble/services/manager/response/DryRun.java @@ -10,6 +10,10 @@ import com.actiontech.dble.cluster.ClusterHelper; import com.actiontech.dble.cluster.ClusterPathUtil; import com.actiontech.dble.config.*; +import com.actiontech.dble.config.converter.DBConverter; +import com.actiontech.dble.config.converter.SequenceConverter; +import com.actiontech.dble.config.converter.ShardingConverter; +import com.actiontech.dble.config.converter.UserConverter; import com.actiontech.dble.config.model.ClusterConfig; import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.config.model.sharding.SchemaConfig; @@ -71,7 +75,17 @@ public static void execute(ManagerService service) { LOGGER.info("reload config(dry-run): load all xml info start"); ConfigInitializer loader; try { - loader = new ConfigInitializer(false); + //sync json + String userConfig = new UserConverter().userXmlToJson(); + String dbConfig = DBConverter.dbXmlToJson(); + String shardingConfig = new ShardingConverter().shardingXmlToJson(); + String sequenceConfig = null; + if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_ZK_GLOBAL_INCREMENT) { + sequenceConfig = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_FILE_NAME); + } else if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_MYSQL) { + sequenceConfig = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_DB_FILE_NAME); + } + loader = new ConfigInitializer(userConfig, dbConfig, shardingConfig, sequenceConfig); } catch (Exception e) { service.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, e.getMessage()); return; @@ -109,9 +123,9 @@ public static void execute(ManagerService service) { } else { try { if (newSystemVariables.isLowerCaseTableNames()) { - serverConfig.reviseLowerCase(); + serverConfig.reviseLowerCase(loader.getSequenceConfig()); } else { - serverConfig.loadSequence(); + serverConfig.loadSequence(loader.getSequenceConfig()); serverConfig.selfChecking0(); } //table exists check ,if the vars can not be touch ,the table check has no meaning diff --git a/src/main/java/com/actiontech/dble/services/manager/response/ReloadConfig.java b/src/main/java/com/actiontech/dble/services/manager/response/ReloadConfig.java index 86fb008cd2..787896fbb5 100644 --- a/src/main/java/com/actiontech/dble/services/manager/response/ReloadConfig.java +++ b/src/main/java/com/actiontech/dble/services/manager/response/ReloadConfig.java @@ -10,14 +10,9 @@ import com.actiontech.dble.backend.datasource.PhysicalDbGroupDiff; import com.actiontech.dble.backend.datasource.ShardingNode; import com.actiontech.dble.btrace.provider.ClusterDelayProvider; -import com.actiontech.dble.cluster.ClusterHelper; -import com.actiontech.dble.cluster.ClusterLogic; -import com.actiontech.dble.cluster.ClusterPathUtil; -import com.actiontech.dble.cluster.DistributeLock; +import com.actiontech.dble.cluster.*; import com.actiontech.dble.cluster.values.ConfStatus; -import com.actiontech.dble.config.ConfigInitializer; -import com.actiontech.dble.config.ErrorCode; -import com.actiontech.dble.config.ServerConfig; +import com.actiontech.dble.config.*; import com.actiontech.dble.config.model.ClusterConfig; import com.actiontech.dble.config.model.SystemConfig; import com.actiontech.dble.config.model.sharding.SchemaConfig; @@ -39,6 +34,7 @@ import com.actiontech.dble.singleton.CronScheduler; import com.actiontech.dble.singleton.FrontendUserManager; import com.actiontech.dble.singleton.TraceManager; +import com.actiontech.dble.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,12 +85,16 @@ public static void execute(ManagerService service, final int loadAllMode, boolea private static void reloadWithCluster(ManagerService service, int loadAllMode, boolean returnFlag, ConfStatus confStatus) throws Exception { TraceManager.TraceObject traceObject = TraceManager.serviceTrace(service, "reload-with-cluster"); try { - DistributeLock distributeLock = ClusterHelper.createDistributeLock(ClusterPathUtil.getConfChangeLockPath(), SystemConfig.getInstance().getInstanceName()); - if (!distributeLock.acquire()) { - service.writeErrMessage(ErrorCode.ER_YES, "Other instance is reloading, please try again later."); - return; + DistributeLock distributeLock = null; + if (!confStatus.getStatus().equals(ConfStatus.Status.MANAGER_INSERT) && !confStatus.getStatus().equals(ConfStatus.Status.MANAGER_UPDATE) && + !confStatus.getStatus().equals(ConfStatus.Status.MANAGER_DELETE)) { + distributeLock = ClusterHelper.createDistributeLock(ClusterPathUtil.getConfChangeLockPath(), SystemConfig.getInstance().getInstanceName()); + if (!distributeLock.acquire()) { + service.writeErrMessage(ErrorCode.ER_YES, "Other instance is reloading, please try again later."); + return; + } + LOGGER.info("reload config: added distributeLock " + ClusterPathUtil.getConfChangeLockPath() + ""); } - LOGGER.info("reload config: added distributeLock " + ClusterPathUtil.getConfChangeLockPath() + ""); ClusterDelayProvider.delayAfterReloadLock(); if (!ReloadManager.startReload(TRIGGER_TYPE_COMMAND, confStatus)) { writeErrorResult(service, "Reload status error ,other client or cluster may in reload"); @@ -105,7 +105,14 @@ private static void reloadWithCluster(ManagerService service, int loadAllMode, b lock.writeLock().lock(); try { //step 2 reload the local config file - if (!reloadAll(loadAllMode)) { + boolean reloadResult; + if (confStatus.getStatus().equals(ConfStatus.Status.MANAGER_INSERT) || confStatus.getStatus().equals(ConfStatus.Status.MANAGER_UPDATE) || + confStatus.getStatus().equals(ConfStatus.Status.MANAGER_DELETE)) { + reloadResult = reloadByConfig(loadAllMode, true); + } else { + reloadResult = reloadByLocalXml(loadAllMode); + } + if (!reloadResult) { writeSpecialError(service, "Reload interruputed by others,config should be reload"); return; } @@ -138,7 +145,9 @@ private static void reloadWithCluster(ManagerService service, int loadAllMode, b } finally { lock.writeLock().unlock(); ClusterHelper.cleanPath(ClusterPathUtil.getConfStatusOperatorPath() + SEPARATOR); - distributeLock.release(); + if (distributeLock != null) { + distributeLock.release(); + } } } finally { TraceManager.finishSpan(service, traceObject); @@ -155,10 +164,16 @@ private static void reloadWithoutCluster(ManagerService service, final int loadA writeErrorResult(service, "Reload status error ,other client or cluster may in reload"); return; } - boolean reloadFlag = reloadAll(loadAllMode); - if (reloadFlag && returnFlag) { + boolean reloadResult; + if (confStatus.getStatus().equals(ConfStatus.Status.MANAGER_INSERT) || confStatus.getStatus().equals(ConfStatus.Status.MANAGER_UPDATE) || + confStatus.getStatus().equals(ConfStatus.Status.MANAGER_DELETE)) { + reloadResult = reloadByConfig(loadAllMode, true); + } else { + reloadResult = reloadByLocalXml(loadAllMode); + } + if (reloadResult && returnFlag) { writeOKResult(service); - } else if (!reloadFlag) { + } else if (!reloadResult) { writeSpecialError(service, "Reload interruputed by others,metadata should be reload"); } } finally { @@ -170,7 +185,7 @@ private static void reloadWithoutCluster(ManagerService service, final int loadA private static void writeOKResult(ManagerService service) { if (LOGGER.isInfoEnabled()) { - ReloadLogHelper.info("send ok package to client " + String.valueOf(service), LOGGER); + ReloadLogHelper.info("send ok package to client " + service, LOGGER); } OkPacket ok = new OkPacket(); @@ -203,7 +218,28 @@ private static void writeErrorResult(ManagerService c, String errorMsg) { c.writeErrMessage(ErrorCode.ER_YES, sb); } - public static boolean reloadAll(final int loadAllMode) throws Exception { + @Deprecated + public static boolean reloadByLocalXml(final int loadAllMode) throws Exception { + return reload(loadAllMode, null, null, null, null); + } + + public static boolean reloadByConfig(final int loadAllMode, boolean isWriteToLocal) throws Exception { + String userConfig = DbleTempConfig.getInstance().getUserConfig(); + userConfig = StringUtil.isBlank(userConfig) ? DbleServer.getInstance().getConfig().getUserConfig() : userConfig; + String dbConfig = DbleTempConfig.getInstance().getDbConfig(); + dbConfig = StringUtil.isBlank(dbConfig) ? DbleServer.getInstance().getConfig().getDbConfig() : dbConfig; + String shardingConfig = DbleTempConfig.getInstance().getShardingConfig(); + shardingConfig = StringUtil.isBlank(shardingConfig) ? DbleServer.getInstance().getConfig().getShardingConfig() : shardingConfig; + String sequenceConfig = DbleTempConfig.getInstance().getSequenceConfig(); + sequenceConfig = StringUtil.isBlank(sequenceConfig) ? DbleServer.getInstance().getConfig().getSequenceConfig() : sequenceConfig; + boolean reloadResult = reload(loadAllMode, userConfig, dbConfig, shardingConfig, sequenceConfig); + DbleTempConfig.getInstance().clean(); + //sync json to local + DbleServer.getInstance().getConfig().syncJsonToLocal(isWriteToLocal); + return reloadResult; + } + + private static boolean reload(final int loadAllMode, String userConfig, String dbConfig, String shardingConfig, String sequenceConfig) throws Exception { TraceManager.TraceObject traceObject = TraceManager.threadTrace("self-reload"); try { /* @@ -214,7 +250,11 @@ public static boolean reloadAll(final int loadAllMode) throws Exception { ReloadLogHelper.info("reload config: load all xml info start", LOGGER); ConfigInitializer loader; try { - loader = new ConfigInitializer(false); + if (null == userConfig && null == dbConfig && null == shardingConfig && null == sequenceConfig) { + loader = new ConfigInitializer(false); + } else { + loader = new ConfigInitializer(userConfig, dbConfig, shardingConfig, sequenceConfig); + } } catch (Exception e) { throw new Exception(e); } @@ -269,10 +309,10 @@ private static boolean intelligentReloadAll(int loadAllMode, ConfigInitializer l if (loader.isFullyConfigured()) { if (newSystemVariables.isLowerCaseTableNames()) { ReloadLogHelper.info("reload config: dbGroup's lowerCaseTableNames=1, lower the config properties start", LOGGER); - serverConfig.reviseLowerCase(); + serverConfig.reviseLowerCase(loader.getSequenceConfig()); ReloadLogHelper.info("reload config: dbGroup's lowerCaseTableNames=1, lower the config properties end", LOGGER); } else { - serverConfig.loadSequence(); + serverConfig.loadSequence(loader.getSequenceConfig()); serverConfig.selfChecking0(); } } @@ -307,7 +347,8 @@ private static boolean intelligentReloadAll(int loadAllMode, ConfigInitializer l boolean result; try { result = config.reload(newUsers, newSchemas, newShardingNodes, mergedDbGroups, recycleHosts, newErRelations, - newSystemVariables, loader.isFullyConfigured(), loadAllMode, newBlacklistConfig, newFunctions); + newSystemVariables, loader.isFullyConfigured(), loadAllMode, newBlacklistConfig, newFunctions, + loader.getUserConfig(), loader.getSequenceConfig(), loader.getShardingConfig(), loader.getDbConfig()); CronScheduler.getInstance().init(config.getSchemas()); if (!result) { initFailed(newDbGroups); @@ -378,10 +419,10 @@ private static boolean forceReloadAll(final int loadAllMode, ConfigInitializer l if (loader.isFullyConfigured()) { if (newSystemVariables.isLowerCaseTableNames()) { ReloadLogHelper.info("reload config: dbGroup's lowerCaseTableNames=1, lower the config properties start", LOGGER); - serverConfig.reviseLowerCase(); + serverConfig.reviseLowerCase(loader.getSequenceConfig()); ReloadLogHelper.info("reload config: dbGroup's lowerCaseTableNames=1, lower the config properties end", LOGGER); } else { - serverConfig.loadSequence(); + serverConfig.loadSequence(loader.getSequenceConfig()); serverConfig.selfChecking0(); } } @@ -404,7 +445,8 @@ private static boolean forceReloadAll(final int loadAllMode, ConfigInitializer l boolean result; try { result = config.reload(newUsers, newSchemas, newShardingNodes, newDbGroups, config.getDbGroups(), newErRelations, - newSystemVariables, loader.isFullyConfigured(), loadAllMode, newBlacklistConfig, newFunctions); + newSystemVariables, loader.isFullyConfigured(), loadAllMode, newBlacklistConfig, newFunctions, + loader.getUserConfig(), loader.getSequenceConfig(), loader.getShardingConfig(), loader.getDbConfig()); CronScheduler.getInstance().init(config.getSchemas()); if (!result) { initFailed(newDbGroups); diff --git a/src/main/java/com/actiontech/dble/singleton/SequenceManager.java b/src/main/java/com/actiontech/dble/singleton/SequenceManager.java index 59a91bcf0a..125bf05f04 100644 --- a/src/main/java/com/actiontech/dble/singleton/SequenceManager.java +++ b/src/main/java/com/actiontech/dble/singleton/SequenceManager.java @@ -45,6 +45,10 @@ public static void load(boolean lowerCaseTableNames) { INSTANCE.handler.load(lowerCaseTableNames); } + public static void load(boolean lowerCaseTableNames, String sequenceJson) { + INSTANCE.handler.loadByJson(lowerCaseTableNames, sequenceJson); + } + public static SequenceManager getInstance() { return INSTANCE; } diff --git a/src/main/java/com/actiontech/dble/util/StringUtil.java b/src/main/java/com/actiontech/dble/util/StringUtil.java index afa02e29de..ec71aeee7a 100644 --- a/src/main/java/com/actiontech/dble/util/StringUtil.java +++ b/src/main/java/com/actiontech/dble/util/StringUtil.java @@ -97,6 +97,10 @@ public static boolean isEmpty(String str) { return ((str == null) || (str.length() == 0)); } + public static boolean isBlank(String str) { + return ((str == null) || (str.length() == 0)) || (str.trim().length() == 0); + } + public static byte[] hexString2Bytes(char[] hexString, int offset, int length) { if (hexString == null) { diff --git a/src/test/java/com/actiontech/dble/cluster/ClusterHelpTest.java b/src/test/java/com/actiontech/dble/cluster/ClusterHelpTest.java index 39af720d9f..31478981c8 100644 --- a/src/test/java/com/actiontech/dble/cluster/ClusterHelpTest.java +++ b/src/test/java/com/actiontech/dble/cluster/ClusterHelpTest.java @@ -5,99 +5,156 @@ package com.actiontech.dble.cluster; -import com.actiontech.dble.cluster.zkprocess.entity.DbGroups; -import com.actiontech.dble.cluster.zkprocess.entity.Shardings; -import com.actiontech.dble.cluster.zkprocess.entity.Users; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.Table; -import com.actiontech.dble.cluster.zkprocess.entity.sharding.schema.TableGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.entity.user.User; -import com.actiontech.dble.cluster.zkprocess.entity.user.UserGsonAdapter; -import com.actiontech.dble.cluster.zkprocess.parse.XmlProcessBase; -import com.actiontech.dble.config.util.ConfigUtil; -import com.actiontech.dble.util.ResourceUtil; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import com.actiontech.dble.backend.datasource.PhysicalDbGroup; +import com.actiontech.dble.backend.datasource.PhysicalDbInstance; +import com.actiontech.dble.backend.datasource.ShardingNode; +import com.actiontech.dble.config.ConfigFileName; +import com.actiontech.dble.config.ConfigInitializer; +import com.actiontech.dble.config.converter.DBConverter; +import com.actiontech.dble.config.converter.SequenceConverter; +import com.actiontech.dble.config.converter.ShardingConverter; +import com.actiontech.dble.config.converter.UserConverter; +import com.actiontech.dble.config.model.ClusterConfig; +import com.actiontech.dble.config.model.sharding.SchemaConfig; +import com.actiontech.dble.config.model.sharding.table.*; +import com.actiontech.dble.config.model.user.*; +import com.actiontech.dble.route.function.AbstractPartitionAlgorithm; import org.junit.Assert; import org.junit.Test; -import org.w3c.dom.Document; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.InputStream; -import java.io.StringWriter; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLStreamException; +import java.util.*; public class ClusterHelpTest { - @Test - public void testShardingXml() throws Exception { - String READ_PATH = ClusterPathUtil.LOCAL_WRITE_PATH + "sharding_template.xml"; - String originXml = loadOriginXml(ClusterPathUtil.LOCAL_WRITE_PATH + "sharding.dtd", READ_PATH); - XmlProcessBase xmlProcess = new XmlProcessBase(); - xmlProcess.addParseClass(Shardings.class); - xmlProcess.initJaxbClass(); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Table.class, new TableGsonAdapter()); - Gson gson = gsonBuilder.create(); - String jsonContent = ClusterLogic.parseShardingXmlFileToJson(xmlProcess, gson, READ_PATH); - Shardings newShardingBean = ClusterLogic.parseShardingJsonToBean(gson, jsonContent); - ClusterLogic.writeMapFileAddFunction(newShardingBean.getFunction()); - String newXml = xmlProcess.baseParseToString(newShardingBean, "sharding"); - Assert.assertEquals(originXml.length(), newXml.length()); + UserConverter userConverter = new UserConverter(); + ShardingConverter shardingConverter = new ShardingConverter(); + String sequencePropsToJson = null; + + ConfigInitializer configInitializerByJson = new ConfigInitializer(userConverter.userXmlToJson(), DBConverter.dbXmlToJson(), shardingConverter.shardingXmlToJson(), sequencePropsToJson); + + ConfigInitializer configInitializerByXml = new ConfigInitializer(false); + + public ClusterHelpTest() throws JAXBException, XMLStreamException { + if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_ZK_GLOBAL_INCREMENT) { + sequencePropsToJson = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_FILE_NAME); + } else if (ClusterConfig.getInstance().getSequenceHandlerType() == ClusterConfig.SEQUENCE_HANDLER_MYSQL) { + sequencePropsToJson = SequenceConverter.sequencePropsToJson(ConfigFileName.SEQUENCE_DB_FILE_NAME); + } } @Test - public void testDXml() throws Exception { - String READ_PATH = ClusterPathUtil.LOCAL_WRITE_PATH + "db_template.xml"; - String originXml = loadOriginXml(ClusterPathUtil.LOCAL_WRITE_PATH + "db.dtd", READ_PATH); - XmlProcessBase xmlProcess = new XmlProcessBase(); - xmlProcess.addParseClass(DbGroups.class); - xmlProcess.initJaxbClass(); - Gson gson = new Gson(); - String jsonContent = ClusterLogic.parseDbGroupXmlFileToJson(xmlProcess, gson, READ_PATH); - DbGroups newDbGroupsBean = ClusterLogic.parseDbGroupsJsonToBean(gson, jsonContent); - String newXml = xmlProcess.baseParseToString(newDbGroupsBean, "db"); - Assert.assertEquals(originXml.length(), newXml.length()); - } + public void testShardingXml() { + Map> erTableSetMapByXml = configInitializerByXml.getErRelations(); + Map functionMapByXml = configInitializerByXml.getFunctions(); + Map schemaConfigMapByXml = configInitializerByXml.getSchemas(); + Map shardingNodeMapByXml = configInitializerByXml.getShardingNodes(); + Map> erTableSetMapByJson = configInitializerByJson.getErRelations(); + Map functionMapByJson = configInitializerByJson.getFunctions(); + Map schemaConfigMapByJson = configInitializerByJson.getSchemas(); + Map shardingNodeMapByJson = configInitializerByJson.getShardingNodes(); + + Assert.assertEquals(erTableSetMapByXml.size(), erTableSetMapByJson.size()); + for (Map.Entry> erTableSetEntry : erTableSetMapByXml.entrySet()) { + Set erTableJson = erTableSetMapByJson.get(erTableSetEntry.getKey()); + Assert.assertEquals(erTableJson, erTableSetEntry.getValue()); + } + + Assert.assertEquals(functionMapByXml.size(), functionMapByJson.size()); + for (Map.Entry algorithmEntry : functionMapByXml.entrySet()) { + AbstractPartitionAlgorithm algorithmJson = functionMapByJson.get(algorithmEntry.getKey()); + Assert.assertEquals(algorithmJson, algorithmEntry.getValue()); + } + + + Assert.assertEquals(shardingNodeMapByXml.size(), shardingNodeMapByJson.size()); + for (Map.Entry shardingNodeConfigEntry : shardingNodeMapByXml.entrySet()) { + ShardingNode shardingNodeJson = shardingNodeMapByJson.get(shardingNodeConfigEntry.getKey()); + Assert.assertTrue(shardingNodeJson.equalsBaseInfo(shardingNodeConfigEntry.getValue())); + } + Assert.assertEquals(schemaConfigMapByXml.size(), schemaConfigMapByJson.size()); + for (Map.Entry schemaConfigEntry : schemaConfigMapByXml.entrySet()) { + SchemaConfig schemaConfigJson = schemaConfigMapByJson.get(schemaConfigEntry.getKey()); + Assert.assertTrue(schemaConfigJson.equalsBaseInfo(schemaConfigEntry.getValue())); + Assert.assertEquals(schemaConfigJson.getTables().size(), schemaConfigEntry.getValue().getTables().size()); + for (Map.Entry tableConfigEntry : schemaConfigJson.getTables().entrySet()) { + Assert.assertTrue(tableConfigEntry.getValue().equalsBaseInfo(schemaConfigEntry.getValue().getTables().get(tableConfigEntry.getKey()))); + + BaseTableConfig baseTableConfig = schemaConfigEntry.getValue().getTables().get(tableConfigEntry.getKey()); + if (baseTableConfig instanceof ShardingTableConfig) { + Assert.assertTrue(((ShardingTableConfig) tableConfigEntry.getValue()).equalsBaseInfo((ShardingTableConfig) baseTableConfig)); + } else if (baseTableConfig instanceof ChildTableConfig) { + Assert.assertTrue(((ChildTableConfig) tableConfigEntry.getValue()).equalsBaseInfo((ChildTableConfig) baseTableConfig)); + } else if (baseTableConfig instanceof GlobalTableConfig) { + Assert.assertTrue(((GlobalTableConfig) tableConfigEntry.getValue()).equalsBaseInfo((GlobalTableConfig) baseTableConfig)); + } else { + Assert.assertTrue(tableConfigEntry.getValue().equalsBaseInfo(baseTableConfig)); + } + } + } + + } @Test - public void testUserXml() throws Exception { - String READ_PATH = ClusterPathUtil.LOCAL_WRITE_PATH + "user_template.xml"; - String originXml = loadOriginXml(ClusterPathUtil.LOCAL_WRITE_PATH + "user.dtd", READ_PATH); - XmlProcessBase xmlProcess = new XmlProcessBase(); - xmlProcess.addParseClass(Users.class); - xmlProcess.initJaxbClass(); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(User.class, new UserGsonAdapter()); - Gson gson = gsonBuilder.create(); - String jsonContent = ClusterLogic.parseUserXmlFileToJson(xmlProcess, gson, READ_PATH); - Users newShardingBean = ClusterLogic.parseUserJsonToBean(gson, jsonContent); - String newXml = xmlProcess.baseParseToString(newShardingBean, "user"); - Assert.assertEquals(originXml.length(), newXml.length()); + public void testDXml() { + Map dbGroupsByXml = configInitializerByXml.getDbGroups(); + Assert.assertEquals(dbGroupsByXml.size(), configInitializerByJson.getDbGroups().size()); + for (Map.Entry physicalDbGroupEntry : dbGroupsByXml.entrySet()) { + PhysicalDbGroup physicalDbGroup = configInitializerByJson.getDbGroups().get(physicalDbGroupEntry.getKey()); + Assert.assertTrue(physicalDbGroupEntry.getValue().equalsBaseInfo(physicalDbGroup)); + Assert.assertEquals(physicalDbGroupEntry.getValue().getAllDbInstanceMap().size(), physicalDbGroup.getAllDbInstanceMap().size()); + for (Map.Entry physicalDbInstanceEntry : physicalDbGroupEntry.getValue().getAllDbInstanceMap().entrySet()) { + Assert.assertEquals(physicalDbInstanceEntry.getValue(), physicalDbGroup.getAllDbInstanceMap().get(physicalDbInstanceEntry.getKey())); + } + } + + + for (Map.Entry physicalDbGroupEntry : configInitializerByJson.getDbGroups().entrySet()) { + PhysicalDbGroup physicalDbGroup = dbGroupsByXml.get(physicalDbGroupEntry.getKey()); + Assert.assertTrue(physicalDbGroupEntry.getValue().equalsBaseInfo(physicalDbGroup)); + Assert.assertEquals(physicalDbGroupEntry.getValue().getAllDbInstanceMap().size(), physicalDbGroup.getAllDbInstanceMap().size()); + for (Map.Entry physicalDbInstanceEntry : physicalDbGroupEntry.getValue().getAllDbInstanceMap().entrySet()) { + Assert.assertEquals(physicalDbInstanceEntry.getValue(), physicalDbGroup.getAllDbInstanceMap().get(physicalDbInstanceEntry.getKey())); + } + } } - private String loadOriginXml(String dtdFile, String xmlFile) throws Exception { - InputStream dtd = ResourceUtil.getResourceAsStreamFromRoot(dtdFile); - InputStream xmlStream = ResourceUtil.getResourceAsStreamFromRoot(xmlFile); - Document root = ConfigUtil.getDocument(dtd, xmlStream); - StringWriter sw = new StringWriter(); - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - transformer.transform(new DOMSource(root), new StreamResult(sw)); - String xml = sw.toString(); - xml = xml.replace("", ""); - if (xml.charAt(xml.length() - 1) == '\n') { - xml = xml.substring(0, xml.length() - 1); + @Test + public void testUserXml() { + Map users = configInitializerByXml.getUsers(); + Map blacklistConfig = configInitializerByXml.getBlacklistConfig(); + Map userConfigMap = configInitializerByJson.getUsers(); + Map blackListConfigMap = configInitializerByJson.getBlacklistConfig(); + Assert.assertEquals(users.size(), userConfigMap.size()); + Assert.assertEquals(blacklistConfig, blackListConfigMap); + + for (Map.Entry userConfigEntry : users.entrySet()) { + UserConfig userConfig = userConfigMap.get(userConfigEntry.getKey()); + if (userConfig instanceof ShardingUserConfig) { + Assert.assertTrue(((ShardingUserConfig) userConfigEntry.getValue()).equalsBaseInfo((ShardingUserConfig) userConfig)); + } else if (userConfig instanceof RwSplitUserConfig) { + Assert.assertTrue(((RwSplitUserConfig) userConfigEntry.getValue()).equalsBaseInfo((RwSplitUserConfig) userConfig)); + } else if (userConfig instanceof ManagerUserConfig) { + Assert.assertTrue(((ManagerUserConfig) userConfigEntry.getValue()).equalsBaseInfo((ManagerUserConfig) userConfig)); + } else { + Assert.assertTrue(userConfigEntry.getValue().equalsBaseInfo(userConfig)); + } + } + for (Map.Entry userConfigEntry : userConfigMap.entrySet()) { + UserConfig userConfig = users.get(userConfigEntry.getKey()); + if (userConfig instanceof ShardingUserConfig) { + Assert.assertTrue(((ShardingUserConfig) userConfigEntry.getValue()).equalsBaseInfo((ShardingUserConfig) userConfig)); + } else if (userConfig instanceof RwSplitUserConfig) { + Assert.assertTrue(((RwSplitUserConfig) userConfigEntry.getValue()).equalsBaseInfo((RwSplitUserConfig) userConfig)); + } else if (userConfig instanceof ManagerUserConfig) { + Assert.assertTrue(((ManagerUserConfig) userConfigEntry.getValue()).equalsBaseInfo((ManagerUserConfig) userConfig)); + } else { + Assert.assertTrue(userConfigEntry.getValue().equalsBaseInfo(userConfig)); + } } - String regexPattern = "( )*"; - Pattern pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE); - Matcher matcher = pattern.matcher(xml); - xml = matcher.replaceAll(""); - return xml.replaceAll("((\r\n)|\n)[\\s\t ]*(\\1)+", "$1").replaceAll("^((\r\n)|\n)", "").replaceAll("(\r)", ""); + } }