Merge config files to iotdb-system.properties (#12570)

Co-authored-by: Haonan <hhaonan@outlook.com>

.github/workflows/multi-language-client.yml

@@ -97,7 +97,10 @@ jobs:
       - name: Integration test
         shell: bash
         run: |
-          cd iotdb-client/client-go/ && make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo
+          cd iotdb-client 
+          git clone https://github.com/apache/iotdb-client-go.git 
+          cd iotdb-client-go 
+          make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo
 
   python:
     runs-on: ubuntu-latest

docker/src/main/DockerCompose/entrypoint.sh

@@ -36,7 +36,7 @@ function on_stop(){
 trap 'on_stop' SIGTERM SIGKILL SIGQUIT
 
 
-replace-conf-from-env.sh ${start_what}
+replace-conf-from-env.sh
 
 case "$1" in
    datanode)

docker/src/main/DockerCompose/replace-conf-from-env.sh

@@ -19,6 +19,7 @@
 #
 
 conf_path=${IOTDB_HOME}/conf
+target_files="iotdb-system.properties"
 
 function process_single(){
 	local key_value="$1"
@@ -48,17 +49,5 @@ function replace_configs(){
   done
 }
 
-case "$1" in
-  confignode)
-    target_files="iotdb-common.properties iotdb-confignode.properties"
-    ;;
-  datanode)
-    target_files="iotdb-common.properties iotdb-datanode.properties"
-    ;;
-  all)
-    target_files="iotdb-common.properties iotdb-confignode.properties iotdb-datanode.properties"
-    ;;
-esac
-
 replace_configs
 

docker/src/main/Dockerfile-1c1d

@@ -30,13 +30,13 @@ RUN apt update \
   && rm /apache-iotdb-*-bin.zip \
   && mv /apache-iotdb-* /iotdb \
   && mv /start-1c1d.sh /iotdb/sbin \
-  && sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-datanode.properties \
-  && sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-confignode.properties \
-  && sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-confignode.properties \
-  && sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-datanode.properties \
-  && sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \
-  && sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \
-  && sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \
+  && sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \
+  && sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \
+  && sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \
+  && sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \
+  && sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \
+  && sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \
+  && sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \
   && apt remove unzip -y \
   && apt autoremove -y \
   && apt purge --auto-remove -y \

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java

@@ -118,6 +118,7 @@ public class ClusterConstant {
   public static final String CONFIG_NODE_PROPERTIES_FILE = "iotdb-confignode.properties";
   public static final String DATA_NODE_PROPERTIES_FILE = "iotdb-datanode.properties";
   public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties";
+  public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties";
 
   public static final String SYSTEM_PROPERTIES_FILE = "system.properties";
   public static final String CONFIG_NODE_SYSTEM_PROPERTIES_FILE = "confignode-system.properties";

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java

@@ -97,7 +97,7 @@ public final void updateProperties(@Nullable String filePath) throws IOException
    * @throws IOException if properties storage failed.
    */
   public final void persistent(String filePath) throws IOException {
-    try (FileWriter confOutput = new FileWriter(filePath)) {
+    try (FileWriter confOutput = new FileWriter(filePath, true)) {
       properties.store(confOutput, null);
     }
   }

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java

@@ -301,8 +301,8 @@ public final void changeConfig(
       outputNodeConfig.updateProperties(immutableNodeProperties);
 
       // Persistent
-      outputCommonConfig.persistent(getTargetCommonConfigPath());
-      outputNodeConfig.persistent(getTargetNodeConfigPath());
+      outputCommonConfig.persistent(getSystemConfigPath());
+      outputNodeConfig.persistent(getSystemConfigPath());
     } catch (IOException ex) {
       throw new AssertionError("Change the config of node failed. " + ex);
     }
@@ -662,9 +662,7 @@ private String getKillPoints() {
 
   protected abstract void renameFile();
 
-  protected abstract String getTargetNodeConfigPath();
-
-  protected abstract String getTargetCommonConfigPath();
+  protected abstract String getSystemConfigPath();
 
   /** Return the node config file path specified through system variable */
   protected abstract String getDefaultNodeConfigPath();

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java

@@ -33,17 +33,16 @@
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_CONSENSUS_DIR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_METRIC_IOTDB_REPORTER_HOST;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_SYSTEM_DIR;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_SYSTEM_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_PROPERTIES;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET;
@@ -90,13 +89,8 @@ public ConfigNodeWrapper(
   }
 
   @Override
-  protected String getTargetNodeConfigPath() {
-    return workDirFilePath("conf", CONFIG_NODE_PROPERTIES_FILE);
-  }
-
-  @Override
-  protected String getTargetCommonConfigPath() {
-    return workDirFilePath("conf", COMMON_PROPERTIES_FILE);
+  protected String getSystemConfigPath() {
+    return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE);
   }
 
   @Override

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java

@@ -29,13 +29,11 @@
 import java.util.List;
 
 import static org.apache.iotdb.consensus.ConsensusFactory.SIMPLE_CONSENSUS;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_INIT_HEAP_SIZE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_DIRECT_MEMORY_SIZE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES;
@@ -53,6 +51,7 @@
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_SYSTEM_DIR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_TRACING_DIR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_WAL_DIRS;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAIN_CLASS_NAME;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAX_TSBLOCK_SIZE_IN_BYTES;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_HOST;
@@ -124,13 +123,8 @@ public DataNodeWrapper(
   }
 
   @Override
-  protected String getTargetNodeConfigPath() {
-    return workDirFilePath("conf", DATA_NODE_PROPERTIES_FILE);
-  }
-
-  @Override
-  protected String getTargetCommonConfigPath() {
-    return workDirFilePath("conf", COMMON_PROPERTIES_FILE);
+  protected String getSystemConfigPath() {
+    return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE);
   }
 
   @Override

iotdb-client/cli/src/assembly/resources/tools/backup.bat

@@ -38,7 +38,12 @@ IF EXIST "%IOTDB_CONF%\datanode-env.bat" (
   echo Can't find datanode-env.bat
 )
 
-IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" (
+IF EXIST "%IOTDB_CONF%\iotdb-system.properties" (
+  for /f  "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port"
+    "%IOTDB_CONF%\iotdb-system.properties"') do (
+      set dn_rpc_port=%%i
+  )
+) ELSE IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" (
   for /f  "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port"
     "%IOTDB_CONF%\iotdb-datanode.properties"') do (
       set dn_rpc_port=%%i

iotdb-client/cli/src/assembly/resources/tools/backup.sh

@@ -70,7 +70,12 @@ get_properties_value() {
     local file_name=$1
     local property_name=$2
     local default_value=$3
-    local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties")
+    if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then
+            local file_path="${IOTDB_HOME}/conf/iotdb-system.properties"
+    else
+            local file_path="${IOTDB_HOME}/conf/${file_name}.properties"
+    fi
+    local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${file_path}")
     if [ -z "$property_value" ]; then
             property_value="$default_value"
     fi

iotdb-client/cli/src/assembly/resources/tools/collect-info.bat

@@ -46,7 +46,11 @@ set "passwd_param=root"
 set "host_param=127.0.0.1"
 set "port_param=6667"
 
-set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties"
+if exist "%IOTDB_HOME%\conf\iotdb-system.properties" (
+  set "properties_file=%IOTDB_HOME%\conf\iotdb-system.properties"
+) else (
+  set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties"
+)
 set "key=dn_data_dirs"
 
 for /f "usebackq tokens=1,* delims==" %%a in ("%properties_file%") do (

iotdb-client/cli/src/assembly/resources/tools/collect-info.sh

@@ -85,7 +85,11 @@ choose_unit() {
     echo "$unit"
 }
 
-properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties"
+if [ -f "$IOTDB_HOME/conf/iotdb-system.properties" ]; then
+  properties_file="$IOTDB_HOME/conf/iotdb-system.properties"
+else
+  properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties"
+fi
 data_dir_key="dn_data_dirs"
 value=$(get_property_value "$properties_file" "$data_dir_key")
 if [ -n "$value" ]; then

iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java

@@ -19,8 +19,8 @@
 
 package org.apache.iotdb.tool;
 
+import org.apache.iotdb.commons.conf.CommonConfig;
 import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 
 import org.slf4j.Logger;
@@ -62,8 +62,6 @@ public class IoTDBDataBackTool {
   static String targetWalDirParam = "";
   static String remoteDnDataDir = "";
 
-  static String DATA_NODE_CONF_NAME = IoTDBConfig.CONFIG_NAME;
-  static String CONFIG_NODE_CONF_NAME = "iotdb-confignode.properties";
   static AtomicInteger fileCount = new AtomicInteger(0);
   static AtomicInteger targetFileCount = new AtomicInteger(0);
   static AtomicInteger processFileCount = new AtomicInteger(0);
@@ -177,10 +175,10 @@ public static void main(String[] args) {
     System.setProperty("IOTDB_HOME", System.getenv("IOTDB_HOME"));
     argsParse(args);
     File sourceDir = new File(sourcePath);
-    Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME);
-    initDataNodeProperties(dataProperties);
-    Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME);
-    initConfigNodeProperties(configProperties);
+
+    Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
+    initDataNodeProperties(properties);
+    initConfigNodeProperties(properties);
 
     StringBuilder targetDirString = new StringBuilder();
     Map<String, String> copyMap = new HashMap<>();
@@ -215,7 +213,7 @@ public static void main(String[] args) {
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + CONFIG_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             cnMapProperties);
       } else if (nodeTypeParam.equalsIgnoreCase("datanode")) {
         countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties);
@@ -237,7 +235,7 @@ public static void main(String[] args) {
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + DATA_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             dnMapProperties);
       } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) {
         countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties);
@@ -260,14 +258,14 @@ public static void main(String[] args) {
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + CONFIG_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             cnMapProperties);
         propertiesFileUpdate(
             targetDirString.toString()
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + DATA_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             dnMapProperties);
       }
 
@@ -293,7 +291,7 @@ public static void main(String[] args) {
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + CONFIG_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             cnMapProperties);
       } else if (nodeTypeParam.equalsIgnoreCase("datanode")) {
         countNodeBack(targetDirString.toString(), copyMap);
@@ -313,7 +311,7 @@ public static void main(String[] args) {
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + DATA_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             dnMapProperties);
       } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) {
         countNodeBack(targetDirString.toString(), copyMap);
@@ -335,14 +333,14 @@ public static void main(String[] args) {
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + CONFIG_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             cnMapProperties);
         propertiesFileUpdate(
             targetDirString.toString()
                 + File.separatorChar
                 + "conf"
                 + File.separatorChar
-                + DATA_NODE_CONF_NAME,
+                + CommonConfig.SYSTEM_CONFIG_NAME,
             dnMapProperties);
       }
     }
@@ -444,10 +442,9 @@ private static void ioTDBDataBack(
 
   private static void countNodeBack(String targetDirString, Map<String, String> copyMap) {
     File sourceDir = new File(sourcePath);
-    Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME);
-    initDataNodeProperties(dataProperties);
-    Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME);
-    initConfigNodeProperties(configProperties);
+    Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
+    initDataNodeProperties(properties);
+    initConfigNodeProperties(properties);
 
     copyMap.put(
         sourceDir.getAbsolutePath() + File.separatorChar + ".env",
@@ -495,7 +492,7 @@ private static void countDataNodeFile(
       Map<String, String> copyMap,
       Map<String, String> dnDataDirsMap,
       Map<String, String> dnMapProperties) {
-    Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME);
+    Properties dataProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
     initDataNodeProperties(dataProperties);
 
     String dnSystemDir = dataProperties.getProperty("dn_system_dir");
@@ -593,7 +590,7 @@ private static String isObjectStorage(String dnDataDirs) {
 
   private static void countConfigNodeFile(
       String targetDirString, Map<String, String> copyMap, Map<String, String> cnMapProperties) {
-    Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME);
+    Properties configProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
     initConfigNodeProperties(configProperties);
 
     String bakCnSystemDir = targetDirString + File.separatorChar + DEFAULT_CN_SYSTEM_DIR;

iotdb-core/confignode/src/assembly/confignode.xml

@@ -44,8 +44,8 @@
     </fileSets>
     <files>
         <file>
-            <source>${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties</source>
-            <destName>conf/iotdb-common.properties</destName>
+            <source>${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties</source>
+            <destName>conf/iotdb-system.properties</destName>
         </file>
         <file>
             <source>${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh</source>