diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml index fcf9c23e87be..b66fa4ed9f56 100644 --- a/.github/workflows/multi-language-client.yml +++ b/.github/workflows/multi-language-client.yml @@ -97,7 +97,10 @@ jobs: - name: Integration test shell: bash run: | - cd iotdb-client/client-go/ && make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo + cd iotdb-client + git clone https://github.com/apache/iotdb-client-go.git + cd iotdb-client-go + make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo python: runs-on: ubuntu-latest diff --git a/docker/src/main/DockerCompose/entrypoint.sh b/docker/src/main/DockerCompose/entrypoint.sh index 55be9899b003..72beea010eeb 100755 --- a/docker/src/main/DockerCompose/entrypoint.sh +++ b/docker/src/main/DockerCompose/entrypoint.sh @@ -36,7 +36,7 @@ function on_stop(){ trap 'on_stop' SIGTERM SIGKILL SIGQUIT -replace-conf-from-env.sh ${start_what} +replace-conf-from-env.sh case "$1" in datanode) diff --git a/docker/src/main/DockerCompose/replace-conf-from-env.sh b/docker/src/main/DockerCompose/replace-conf-from-env.sh index 93be28bc2425..887faf1bdcd6 100755 --- a/docker/src/main/DockerCompose/replace-conf-from-env.sh +++ b/docker/src/main/DockerCompose/replace-conf-from-env.sh @@ -19,6 +19,7 @@ # conf_path=${IOTDB_HOME}/conf +target_files="iotdb-system.properties" function process_single(){ local key_value="$1" @@ -48,17 +49,5 @@ function replace_configs(){ done } -case "$1" in - confignode) - target_files="iotdb-common.properties iotdb-confignode.properties" - ;; - datanode) - target_files="iotdb-common.properties iotdb-datanode.properties" - ;; - all) - target_files="iotdb-common.properties iotdb-confignode.properties iotdb-datanode.properties" - ;; -esac - replace_configs diff --git a/docker/src/main/Dockerfile-1c1d b/docker/src/main/Dockerfile-1c1d index 9aacd0aa0ca9..71548de08b78 100644 --- a/docker/src/main/Dockerfile-1c1d +++ b/docker/src/main/Dockerfile-1c1d @@ -30,13 +30,13 @@ RUN apt update \ && rm /apache-iotdb-*-bin.zip \ && mv /apache-iotdb-* /iotdb \ && mv /start-1c1d.sh /iotdb/sbin \ - && sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-datanode.properties \ - && sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-confignode.properties \ - && sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-confignode.properties \ - && sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-datanode.properties \ - && sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \ - && sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \ - && sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \ + && sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \ && apt remove unzip -y \ && apt autoremove -y \ && apt purge --auto-remove -y \ diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index b827d01e504f..b1eff829299e 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -118,6 +118,7 @@ public class ClusterConstant { public static final String CONFIG_NODE_PROPERTIES_FILE = "iotdb-confignode.properties"; public static final String DATA_NODE_PROPERTIES_FILE = "iotdb-datanode.properties"; public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties"; + public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties"; public static final String SYSTEM_PROPERTIES_FILE = "system.properties"; public static final String CONFIG_NODE_SYSTEM_PROPERTIES_FILE = "confignode-system.properties"; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java index 04e5dae88724..473d2f1b64c7 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java @@ -97,7 +97,7 @@ public final void updateProperties(@Nullable String filePath) throws IOException * @throws IOException if properties storage failed. */ public final void persistent(String filePath) throws IOException { - try (FileWriter confOutput = new FileWriter(filePath)) { + try (FileWriter confOutput = new FileWriter(filePath, true)) { properties.store(confOutput, null); } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index ab14613fc41c..fa5447bc5b07 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -301,8 +301,8 @@ public final void changeConfig( outputNodeConfig.updateProperties(immutableNodeProperties); // Persistent - outputCommonConfig.persistent(getTargetCommonConfigPath()); - outputNodeConfig.persistent(getTargetNodeConfigPath()); + outputCommonConfig.persistent(getSystemConfigPath()); + outputNodeConfig.persistent(getSystemConfigPath()); } catch (IOException ex) { throw new AssertionError("Change the config of node failed. " + ex); } @@ -662,9 +662,7 @@ private String getKillPoints() { protected abstract void renameFile(); - protected abstract String getTargetNodeConfigPath(); - - protected abstract String getTargetCommonConfigPath(); + protected abstract String getSystemConfigPath(); /** Return the node config file path specified through system variable */ protected abstract String getDefaultNodeConfigPath(); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index 8cbffe060607..a457149f5985 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -33,17 +33,16 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_CONSENSUS_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_METRIC_IOTDB_REPORTER_HOST; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_SYSTEM_DIR; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_PROPERTIES; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET; @@ -90,13 +89,8 @@ public ConfigNodeWrapper( } @Override - protected String getTargetNodeConfigPath() { - return workDirFilePath("conf", CONFIG_NODE_PROPERTIES_FILE); - } - - @Override - protected String getTargetCommonConfigPath() { - return workDirFilePath("conf", COMMON_PROPERTIES_FILE); + protected String getSystemConfigPath() { + return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } @Override diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index e6d16f9860b3..1135f5cd055c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -29,13 +29,11 @@ import java.util.List; import static org.apache.iotdb.consensus.ConsensusFactory.SIMPLE_CONSENSUS; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_INIT_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_DIRECT_MEMORY_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES; @@ -53,6 +51,7 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_SYSTEM_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_TRACING_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_WAL_DIRS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAIN_CLASS_NAME; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAX_TSBLOCK_SIZE_IN_BYTES; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_HOST; @@ -124,13 +123,8 @@ public DataNodeWrapper( } @Override - protected String getTargetNodeConfigPath() { - return workDirFilePath("conf", DATA_NODE_PROPERTIES_FILE); - } - - @Override - protected String getTargetCommonConfigPath() { - return workDirFilePath("conf", COMMON_PROPERTIES_FILE); + protected String getSystemConfigPath() { + return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } @Override diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.bat b/iotdb-client/cli/src/assembly/resources/tools/backup.bat index 13178b7f9bb4..01be45f9ca01 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.bat @@ -38,7 +38,12 @@ IF EXIST "%IOTDB_CONF%\datanode-env.bat" ( echo Can't find datanode-env.bat ) -IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" + "%IOTDB_CONF%\iotdb-system.properties"') do ( + set dn_rpc_port=%%i + ) +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" "%IOTDB_CONF%\iotdb-datanode.properties"') do ( set dn_rpc_port=%%i diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.sh b/iotdb-client/cli/src/assembly/resources/tools/backup.sh index 43a6f171371f..7ef5cdc5777f 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.sh @@ -70,7 +70,12 @@ get_properties_value() { local file_name=$1 local property_name=$2 local default_value=$3 - local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties") + if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + local file_path="${IOTDB_HOME}/conf/iotdb-system.properties" + else + local file_path="${IOTDB_HOME}/conf/${file_name}.properties" + fi + local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${file_path}") if [ -z "$property_value" ]; then property_value="$default_value" fi diff --git a/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat b/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat index 0c87d0cba994..246228b9187e 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat @@ -46,7 +46,11 @@ set "passwd_param=root" set "host_param=127.0.0.1" set "port_param=6667" -set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-system.properties" +) else ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +) set "key=dn_data_dirs" for /f "usebackq tokens=1,* delims==" %%a in ("%properties_file%") do ( diff --git a/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh b/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh index a01714b289d7..1926309374a8 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh @@ -85,7 +85,11 @@ choose_unit() { echo "$unit" } -properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties" +if [ -f "$IOTDB_HOME/conf/iotdb-system.properties" ]; then + properties_file="$IOTDB_HOME/conf/iotdb-system.properties" +else + properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties" +fi data_dir_key="dn_data_dirs" value=$(get_property_value "$properties_file" "$data_dir_key") if [ -n "$value" ]; then diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index c9d9df6aae6a..b84a2e30a4cc 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -19,8 +19,8 @@ package org.apache.iotdb.tool; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; @@ -62,8 +62,6 @@ public class IoTDBDataBackTool { static String targetWalDirParam = ""; static String remoteDnDataDir = ""; - static String DATA_NODE_CONF_NAME = IoTDBConfig.CONFIG_NAME; - static String CONFIG_NODE_CONF_NAME = "iotdb-confignode.properties"; static AtomicInteger fileCount = new AtomicInteger(0); static AtomicInteger targetFileCount = new AtomicInteger(0); static AtomicInteger processFileCount = new AtomicInteger(0); @@ -177,10 +175,10 @@ public static void main(String[] args) { System.setProperty("IOTDB_HOME", System.getenv("IOTDB_HOME")); argsParse(args); File sourceDir = new File(sourcePath); - Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); - initDataNodeProperties(dataProperties); - Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME); - initConfigNodeProperties(configProperties); + + Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); + initDataNodeProperties(properties); + initConfigNodeProperties(properties); StringBuilder targetDirString = new StringBuilder(); Map copyMap = new HashMap<>(); @@ -215,7 +213,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("datanode")) { countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); @@ -237,7 +235,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) { countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); @@ -260,14 +258,14 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); propertiesFileUpdate( targetDirString.toString() + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } @@ -293,7 +291,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("datanode")) { countNodeBack(targetDirString.toString(), copyMap); @@ -313,7 +311,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) { countNodeBack(targetDirString.toString(), copyMap); @@ -335,14 +333,14 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); propertiesFileUpdate( targetDirString.toString() + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } } @@ -444,10 +442,9 @@ private static void ioTDBDataBack( private static void countNodeBack(String targetDirString, Map copyMap) { File sourceDir = new File(sourcePath); - Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); - initDataNodeProperties(dataProperties); - Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME); - initConfigNodeProperties(configProperties); + Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); + initDataNodeProperties(properties); + initConfigNodeProperties(properties); copyMap.put( sourceDir.getAbsolutePath() + File.separatorChar + ".env", @@ -495,7 +492,7 @@ private static void countDataNodeFile( Map copyMap, Map dnDataDirsMap, Map dnMapProperties) { - Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); + Properties dataProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); initDataNodeProperties(dataProperties); String dnSystemDir = dataProperties.getProperty("dn_system_dir"); @@ -593,7 +590,7 @@ private static String isObjectStorage(String dnDataDirs) { private static void countConfigNodeFile( String targetDirString, Map copyMap, Map cnMapProperties) { - Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME); + Properties configProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); initConfigNodeProperties(configProperties); String bakCnSystemDir = targetDirString + File.separatorChar + DEFAULT_CN_SYSTEM_DIR; diff --git a/iotdb-core/confignode/src/assembly/confignode.xml b/iotdb-core/confignode/src/assembly/confignode.xml index 7e858de63a04..242e3fb810d0 100644 --- a/iotdb-core/confignode/src/assembly/confignode.xml +++ b/iotdb-core/confignode/src/assembly/confignode.xml @@ -44,8 +44,8 @@ - ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties - conf/iotdb-common.properties + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties + conf/iotdb-system.properties ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh diff --git a/iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties b/iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties deleted file mode 100644 index 8257cc040295..000000000000 --- a/iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties +++ /dev/null @@ -1,147 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -#################### -### Config Node RPC Configuration -#################### - -# Used for RPC communication inside cluster. -# Could set 127.0.0.1(for local test) or ipv4 address. -# Datatype: String -cn_internal_address=127.0.0.1 - -# Used for RPC communication inside cluster. -# Datatype: int -cn_internal_port=10710 - -# Used for consensus communication among ConfigNodes inside cluster. -# Datatype: int -cn_consensus_port=10720 - -#################### -### Seed ConfigNode -#################### - -# For the first ConfigNode to start, cn_seed_config_node points to its own cn_internal_address:cn_internal_port. -# For other ConfigNodes that to join the cluster, cn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. -# Note: After this ConfigNode successfully joins the cluster for the first time, this parameter is no longer used. -# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. -# Format: address:port e.g. 127.0.0.1:10710 -# Datatype: String -cn_seed_config_node=127.0.0.1:10710 - -#################### -### Directory configuration -#################### - -# system dir -# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/system). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# cn_system_dir=data\\confignode\\system -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# cn_system_dir=data/confignode/system - -# consensus dir -# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/consensus). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. -# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# cn_consensus_dir=data\\confignode\\consensus -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# cn_consensus_dir=data/confignode/consensus - -# pipe_receiver_file_dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${cn_system_dir}/pipe/receiver). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If pipe_receiver_file_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dir=data\\confignode\\system\\pipe\\receiver -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dir=data/confignode/system/pipe/receiver - -#################### -### thrift rpc configuration -#################### - -# this feature is under development, set this as false before it is done. -# Datatype: boolean -# cn_rpc_thrift_compression_enable=false - -# if true, a snappy based compression method will be called before sending data by the network -# Datatype: boolean -# this feature is under development, set this as false before it is done. -# cn_rpc_advanced_compression_enable=false - -# Datatype: int -# cn_rpc_max_concurrent_client_num=65535 - -# thrift max frame size, 512MB by default -# Datatype: int -# cn_thrift_max_frame_size=536870912 - -# thrift init buffer size -# Datatype: int -# cn_thrift_init_buffer_size=1024 - -# Thrift socket and connection timeout between raft nodes, in milliseconds. -# Datatype: int -# cn_connection_timeout_ms=60000 - -# selector thread (TAsyncClientManager) nums for async thread in a clientManager -# Datatype: int -# cn_selector_thread_nums_of_client_manager=1 - -# The maximum number of clients that can be allocated for a node in a clientManager. -# when the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked -# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. -# Datatype: int -# cn_max_client_count_for_each_node_in_client_manager=300 - -#################### -### Metric Configuration -#################### - -# The reporters of metric module to report metrics -# If there are more than one reporter, please separate them by commas ",". -# Options: [JMX, PROMETHEUS] -# Datatype: String -# cn_metric_reporter_list= - -# The level of metric module -# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] -# Datatype: String -# cn_metric_level=CORE - -# The period of async collection of some metrics in second -# Datatype: int -# cn_metric_async_collect_period=5 - -# The port of prometheus reporter of metric module -# Datatype: int -# cn_metric_prometheus_reporter_port=9091 \ No newline at end of file diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat index 532c79933051..98399a9990bf 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat @@ -83,26 +83,30 @@ IF EXIST "%CONFIGNODE_CONF%\confignode-env.bat" ( ) @REM CHECK THE PORT USAGES -IF EXIST "%CONFIGNODE_CONF%\iotdb-confignode.properties" ( +@REM SET CONFIG FILE +IF EXIST "%CONFIGNODE_CONF%\iotdb-system.properties" ( + set CONFIG_FILE="%CONFIGNODE_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%CONFIGNODE_HOME%\conf\iotdb-system.properties" ( + set CONFIG_FILE="%CONFIGNODE_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%CONFIGNODE_CONF%\iotdb-confignode.properties" ( + set CONFIG_FILE="%CONFIGNODE_CONF%\iotdb-confignode.properties" +) ELSE IF EXIST "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties" ( + set CONFIG_FILE="%CONFIGNODE_HOME%\conf\iotdb-confignode.properties" +) ELSE ( + set CONFIG_FILE= +) + +IF DEFINED CONFIG_FILE ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - "%CONFIGNODE_CONF%\iotdb-confignode.properties"') do ( + "%CONFIG_FILE%"') do ( set cn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - "%CONFIGNODE_CONF%\iotdb-confignode.properties"') do ( + "%CONFIG_FILE%"') do ( set cn_consensus_port=%%i ) -) ELSE IF EXIST "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties"') do ( - set cn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties"') do ( - set cn_consensus_port=%%i - ) ) ELSE ( - echo "Can't find iotdb-confignode.properties, check the default ports" + echo "Can't find iotdb-system.properties or iotdb-confignode.properties, check the default ports" set cn_internal_port=10710 set cn_consensus_port=10720 ) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index c82a095ec1fa..e69764266887 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -22,15 +22,21 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ +IF EXIST "%superior%\conf\iotdb-system.properties" ( + set config_file="%superior_dir%\conf\iotdb-system.properties" +) ELSE ( + set config_file="%superior_dir%\conf\iotdb-confignode.properties" +) + for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" -"%superior_dir%\conf\iotdb-confignode.properties"') do ( +"%config_file%"') do ( set cn_internal_port=%%i ) echo "check whether the cn_internal_port is used..., port is %cn_internal_port%" for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" -"%superior_dir%\conf\iotdb-confignode.properties"') do ( +"%config_file%"') do ( set cn_internal_address=%%i ) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh index c9467faec55f..5604afd3d32f 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh @@ -19,7 +19,13 @@ # CONFIGNODE_CONF="$(dirname "$0")/../conf" -cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) + +if [ -f "${CONFIGNODE_CONF}/iotdb-system.properties" ]; then + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) +else + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) +fi + echo Check whether the internal_port is used..., port is "$cn_internal_port" diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java index ca521e3a73c9..5724eb1862fe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java @@ -25,7 +25,6 @@ public class ConfigNodeConstant { public static final String CONFIGNODE_CONF = "CONFIGNODE_CONF"; public static final String CONFIGNODE_HOME = "CONFIGNODE_HOME"; - public static final String CONF_FILE_NAME = "iotdb-confignode.properties"; public static final String SYSTEM_FILE_NAME = "confignode-system.properties"; public static final String CONFIGNODE_PACKAGE = "org.apache.iotdb.confignode.service"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index ee9621d2393c..ce713b2e0e89 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -36,7 +37,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -54,6 +54,19 @@ public class ConfigNodeDescriptor { private final ConfigNodeConfig conf = new ConfigNodeConfig(); + static { + ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); + URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); + URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); + URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); + try { + updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + } catch (Exception e) { + LOGGER.error("Failed to update config file", e); + } + } + private ConfigNodeDescriptor() { loadProps(); } @@ -67,7 +80,7 @@ public ConfigNodeConfig getConf() { * * @return url object if location exit, otherwise null. */ - public URL getPropsUrl(String configFileName) { + public static URL getPropsUrl(String configFileName) { // Check if a config-directory was specified first. String urlString = System.getProperty(ConfigNodeConstant.CONFIGNODE_CONF, null); // If it wasn't, check if a home directory was provided @@ -102,37 +115,12 @@ else if (!urlString.endsWith(".properties")) { } private void loadProps() { - URL url = getPropsUrl(CommonConfig.CONFIG_NAME); Properties commonProperties = new Properties(); - if (url != null) { - try (InputStream inputStream = url.openStream()) { - - LOGGER.info("Start to read config file {}", url); - commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - - } catch (FileNotFoundException e) { - LOGGER.error("Fail to find config file {}, reject ConfigNode startup.", url, e); - System.exit(-1); - } catch (IOException e) { - LOGGER.error("Cannot load config file, reject ConfigNode startup.", e); - System.exit(-1); - } catch (Exception e) { - LOGGER.error("Incorrect format in config file, reject ConfigNode startup.", e); - System.exit(-1); - } - } else { - LOGGER.warn( - "Couldn't load the configuration {} from any of the known sources.", - CommonConfig.CONFIG_NAME); - } - - url = getPropsUrl(ConfigNodeConstant.CONF_FILE_NAME); + URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url != null) { try (InputStream inputStream = url.openStream()) { LOGGER.info("start reading ConfigNode conf file: {}", url); - Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - commonProperties.putAll(properties); + commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); loadProperties(commonProperties); } catch (IOException | BadNodeUrlException e) { LOGGER.error("Couldn't load ConfigNode conf file, reject ConfigNode startup.", e); @@ -142,7 +130,7 @@ private void loadProps() { commonDescriptor .getConfig() .updatePath(System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null)); - MetricConfigDescriptor.getInstance().loadProps(commonProperties); + MetricConfigDescriptor.getInstance().loadProps(commonProperties, true); MetricConfigDescriptor.getInstance() .getMetricConfig() .updateRpcInstance( @@ -151,7 +139,7 @@ private void loadProps() { } else { LOGGER.warn( "Couldn't load the configuration {} from any of the known sources.", - ConfigNodeConstant.CONF_FILE_NAME); + CommonConfig.SYSTEM_CONFIG_NAME); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java index 697dd65525d9..13cd42393fab 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java @@ -25,8 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeType; -import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.confignode.conf.ConfigNodeConstant; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.rpc.TSStatusCode; @@ -56,11 +55,6 @@ private ClusterNodeStartUtils() { public static TSStatus confirmNodeRegistration( NodeType nodeType, String clusterName, Object nodeLocation, ConfigManager configManager) { - - final String CONF_FILE_NAME = - NodeType.ConfigNode.equals(nodeType) - ? ConfigNodeConstant.CONF_FILE_NAME - : IoTDBConstant.DATA_NODE_CONF_FILE_NAME; TSStatus status = new TSStatus(); /* Reject start if the cluster name is error */ @@ -77,8 +71,8 @@ public static TSStatus confirmNodeRegistration( nodeType.getNodeType(), clusterName, CLUSTER_NAME, - CONF_FILE_NAME, - CONF_FILE_NAME)); + CommonConfig.SYSTEM_CONFIG_NAME, + CommonConfig.SYSTEM_CONFIG_NAME)); return status; } @@ -116,7 +110,7 @@ public static TSStatus confirmNodeRegistration( nodeType.getNodeType(), conflictEndPoints, nodeType.getNodeType(), - CONF_FILE_NAME)); + CommonConfig.SYSTEM_CONFIG_NAME)); return status; } else { /* Accept registration if all TEndPoints aren't conflict */ @@ -130,11 +124,6 @@ public static TSStatus confirmNodeRestart( int nodeId, Object nodeLocation, ConfigManager configManager) { - - final String CONF_FILE_NAME = - NodeType.ConfigNode.equals(nodeType) - ? ConfigNodeConstant.CONF_FILE_NAME - : IoTDBConstant.DATA_NODE_CONF_FILE_NAME; TSStatus status = new TSStatus(); /* Reject restart if the cluster name is error */ @@ -151,8 +140,8 @@ public static TSStatus confirmNodeRestart( nodeType.getNodeType(), clusterName, CLUSTER_NAME, - CONF_FILE_NAME, - CONF_FILE_NAME)); + CommonConfig.SYSTEM_CONFIG_NAME, + CommonConfig.SYSTEM_CONFIG_NAME)); return status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 65ffa13614c3..eb33b0d70eba 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -329,8 +329,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio TEndPoint seedConfigNode = CONF.getSeedConfigNode(); if (seedConfigNode == null) { - LOGGER.error( - "Please set the cn_seed_config_node parameter in iotdb-confignode.properties file."); + LOGGER.error("Please set the cn_seed_config_node parameter in iotdb-system.properties file."); throw new StartupException("The seedConfigNode setting in conf is empty"); } diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties deleted file mode 100644 index ade8959df993..000000000000 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cn_internal_address=127.0.0.1 -cn_internal_port=10710 -cn_consensus_port=10720 -cn_seed_config_node=127.0.0.1:10710 -cn_system_dir=target/confignode1/system -cn_data_dirs=target/confignode1/data -cn_consensus_dir=target/confignode1/consensus - -cn_metric_reporter_list=PROMETHEUS -cn_metric_level=IMPORTANT -cn_metric_async_collect_period=5 -cn_metric_prometheus_reporter_port=9091 \ No newline at end of file diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties similarity index 76% rename from iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties rename to iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties index 8981b21285bb..b396e373f869 100644 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties +++ b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties @@ -17,6 +17,19 @@ # under the License. # +cn_internal_address=127.0.0.1 +cn_internal_port=10710 +cn_consensus_port=10720 +cn_seed_config_node=127.0.0.1:10710 +cn_system_dir=target/confignode1/system +cn_data_dirs=target/confignode1/data +cn_consensus_dir=target/confignode1/consensus + +cn_metric_reporter_list=PROMETHEUS +cn_metric_level=IMPORTANT +cn_metric_async_collect_period=5 +cn_metric_prometheus_reporter_port=9091 + timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus diff --git a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties b/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties deleted file mode 100644 index eabd45e89e91..000000000000 --- a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cn_internal_address=127.0.0.1 -cn_internal_port=10711 -cn_consensus_port=10721 -cn_seed_config_node=127.0.0.1:10710 -cn_system_dir=target/confignode2/system -cn_data_dirs=target/confignode2/data -cn_consensus_dir=target/confignode2/consensus - -cn_metric_reporter_list=PROMETHEUS -cn_metric_level=IMPORTANT -cn_metric_async_collect_period=5 -cn_metric_prometheus_reporter_port=9093 \ No newline at end of file diff --git a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-system.properties similarity index 76% rename from iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-common.properties rename to iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-system.properties index a9789fedabbe..ef3ba921f96c 100644 --- a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-common.properties +++ b/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-system.properties @@ -17,6 +17,19 @@ # under the License. # +cn_internal_address=127.0.0.1 +cn_internal_port=10711 +cn_consensus_port=10721 +cn_seed_config_node=127.0.0.1:10710 +cn_system_dir=target/confignode2/system +cn_data_dirs=target/confignode2/data +cn_consensus_dir=target/confignode2/consensus + +cn_metric_reporter_list=PROMETHEUS +cn_metric_level=IMPORTANT +cn_metric_async_collect_period=5 +cn_metric_prometheus_reporter_port=9093 + timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus diff --git a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties b/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties deleted file mode 100644 index ad706b698ec5..000000000000 --- a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cn_internal_address=127.0.0.1 -cn_internal_port=10712 -cn_consensus_port=10722 -cn_seed_config_node=127.0.0.1:10710 -cn_system_dir=target/confignode3/system -cn_data_dirs=target/confignode3/data -cn_consensus_dir=target/confignode3/consensus - -cn_metric_reporter_list=PROMETHEUS -cn_metric_level=IMPORTANT -cn_metric_async_collect_period=5 -cn_metric_prometheus_reporter_port=9095 \ No newline at end of file diff --git a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-system.properties similarity index 76% rename from iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-common.properties rename to iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-system.properties index 6a95388bd72c..48f99317a2fd 100644 --- a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-common.properties +++ b/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-system.properties @@ -17,6 +17,19 @@ # under the License. # +cn_internal_address=127.0.0.1 +cn_internal_port=10712 +cn_consensus_port=10722 +cn_seed_config_node=127.0.0.1:10710 +cn_system_dir=target/confignode3/system +cn_data_dirs=target/confignode3/data +cn_consensus_dir=target/confignode3/consensus + +cn_metric_reporter_list=PROMETHEUS +cn_metric_level=IMPORTANT +cn_metric_async_collect_period=5 +cn_metric_prometheus_reporter_port=9095 + timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus diff --git a/iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties b/iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties deleted file mode 100644 index 949e34007500..000000000000 --- a/iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties +++ /dev/null @@ -1,316 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -#################### -### Data Node RPC Configuration -#################### - -# Used for connection of IoTDB native clients(Session) -# Could set 127.0.0.1(for local test) or ipv4 address -# Datatype: String -dn_rpc_address=0.0.0.0 - -# Used for connection of IoTDB native clients(Session) -# Bind with dn_rpc_address -# Datatype: int -dn_rpc_port=6667 - -# Used for communication inside cluster. -# could set 127.0.0.1(for local test) or ipv4 address. -# Datatype: String -dn_internal_address=127.0.0.1 - -# Used for communication inside cluster. -# Bind with dn_internal_address -# Datatype: int -dn_internal_port=10730 - -# Port for data exchange among DataNodes inside cluster -# Bind with dn_internal_address -# Datatype: int -dn_mpp_data_exchange_port=10740 - -# port for consensus's communication for schema region inside cluster. -# Bind with dn_internal_address -# Datatype: int -dn_schema_region_consensus_port=10750 - -# port for consensus's communication for data region inside cluster. -# Bind with dn_internal_address -# Datatype: int -dn_data_region_consensus_port=10760 - -# Datatype: long -# The time of data node waiting for the next retry to join into the cluster. -# dn_join_cluster_retry_interval_ms=5000 - -#################### -### SSL Configuration -#################### - -# Does dn_rpc_port enable SSL -# enable_thrift_ssl=false - -# Rest Service enabled SSL -# enable_https=false - -# SSL key store path -# linux e.g. /home/iotdb/server.keystore (absolute path) or server.keystore (relative path) -# windows e.g. C:\\iotdb\\server.keystore (absolute path) or server.keystore (relative path) -# key_store_path= - -# SSL key store password -# key_store_pwd= - -#################### -### Seed ConfigNode -#################### - -# dn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. -# Note: After this DataNode successfully joins the cluster for the first time, this parameter is no longer used. -# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. -# Format: address:port e.g. 127.0.0.1:10710 -# Datatype: String -dn_seed_config_node=127.0.0.1:10710 - -#################### -### Connection Configuration -#################### - -# The maximum session idle time. unit: ms -# Idle sessions are the ones that performs neither query or non-query operations for a period of time -# Set to 0 to disable session timeout -# Datatype: int -# dn_session_timeout_threshold=0 - -# Datatype: boolean -# dn_rpc_thrift_compression_enable=false - -# if true, a snappy based compression method will be called before sending data by the network -# Datatype: boolean -# this feature is under development, set this as false before it is done. -# dn_rpc_advanced_compression_enable=false - -# Datatype: int -# dn_rpc_selector_thread_count=1 - -# Datatype: int -# dn_rpc_min_concurrent_client_num=1 - -# Datatype: int -# dn_rpc_max_concurrent_client_num=65535 - -# thrift max frame size, 512MB by default -# Datatype: int -# dn_thrift_max_frame_size=536870912 - -# thrift init buffer size -# Datatype: int -# dn_thrift_init_buffer_size=1024 - -# Thrift socket and connection timeout between raft nodes, in milliseconds. -# Datatype: int -# dn_connection_timeout_ms=60000 - -# selector thread (TAsyncClientManager) nums for async thread in a clientManager -# Datatype: int -# dn_selector_thread_count_of_client_manager=1 - -# The maximum number of clients that can be allocated for a node in a clientManager. -# When the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked -# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. -# Datatype: int -# dn_max_client_count_for_each_node_in_client_manager=300 - -#################### -### Directory Configuration -#################### - -# system dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/system). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_system_dir=data\\datanode\\system -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_system_dir=data/datanode/system - - -# data dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/data). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If data_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_data_dirs=data\\datanode\\data -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_data_dirs=data/datanode/data - - -# multi_dir_strategy -# The strategy is used to choose a directory from data_dirs for the system to store a new tsfile. -# System provides two strategies to choose from, or user can create his own strategy by extending org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy. -# The info of the two strategies are as follows: -# 1. SequenceStrategy: the system will choose the directory in sequence. -# 2. MaxDiskUsableSpaceFirstStrategy: the system will choose the directory whose disk has the maximum space. -# Set SequenceStrategy or MaxDiskUsableSpaceFirstStrategy to apply the corresponding strategy. -# If this property is unset, system will use SequenceStrategy as default strategy. -# For this property, fully-qualified class name (include package name) and simple class name are both acceptable. -# dn_multi_dir_strategy=SequenceStrategy - -# consensus dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If consensus_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_consensus_dir=data\\datanode\\consensus -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_consensus_dir=data/datanode/consensus - -# wal dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If wal_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_wal_dirs=data\\datanode\\wal -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_wal_dirs=data/datanode/wal - -# tracing dir -# Uncomment following fields to configure the tracing root directory. -# For Windows platform, the index is as follows: -# dn_tracing_dir=datanode\\tracing -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_tracing_dir=datanode/tracing - -# sync dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_sync_dir=data\\datanode\\sync -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_sync_dir=data/datanode/sync - -# sort_tmp_dir -# This property is used to configure the temporary directory for sorting operation. -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If sort_tmp_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# sort_tmp_dir=data\\datanode\\tmp -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# sort_tmp_dir=data/datanode/tmp - -# pipe_receiver_file_dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${dn_system_dir}/pipe/receiver). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If pipe_receiver_file_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dirs=data\\datanode\\system\\pipe\\receiver -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dirs=data/datanode/system/pipe/receiver - -#################### -### Metric Configuration -#################### - -# The reporters of metric module to report metrics -# If there are more than one reporter, please separate them by commas ",". -# Options: [JMX, PROMETHEUS] -# Datatype: String -# dn_metric_reporter_list= - -# The level of metric module -# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] -# Datatype: String -# dn_metric_level=CORE - -# The period of async collection of some metrics in second -# Datatype: int -# dn_metric_async_collect_period=5 - -# The port of prometheus reporter of metric module -# Datatype: int -dn_metric_prometheus_reporter_port=9092 - -# The type of internal reporter in metric module, used for checking flushed point number -# Options: [MEMORY, IOTDB] -# Datatype: String -# dn_metric_internal_reporter_type=MEMORY - -#################### -### REST Service Configuration -#################### - -# Is the REST service enabled -# enable_rest_service=false - -# the binding port of the REST service -# rest_service_port=18080 - -# Whether to display rest service interface information through swagger. eg: http://ip:port/swagger.json -# enable_swagger=false - -# the default row limit to a REST query response when the rowSize parameter is not given in request -# rest_query_default_row_size_limit=10000 - -# the expiration time of the user login information cache (in seconds) -# cache_expire_in_seconds=28800 - -# maximum number of users can be stored in the user login cache. -# cache_max_num=100 - -# init capacity of users can be stored in the user login cache. -# cache_init_num=10 - -# Is client authentication required -# client_auth=false - -# SSL trust store path -# trust_store_path="" - -# SSL trust store password. -# trust_store_pwd="" - -# SSL timeout (in seconds) -# idle_timeout_in_seconds=50000 diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat index 569374a89d26..c6fd64e8fb34 100755 --- a/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat @@ -102,51 +102,43 @@ IF EXIST "%IOTDB_CONF%\datanode-env.bat" ( echo "Can't find datanode-env.bat" ) +@REM SET CONFIG FILE +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIG_FILE="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( + set CONFIG_FILE="%IOTDB_CONF%\iotdb-datanode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( + set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-datanode.properties" +) ELSE ( + set CONFIG_FILE= +) + @REM CHECK THE PORT USAGES -IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( +IF DEFINED CONFIG_FILE ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_rpc_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( - set dn_mpp_data_exchange_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( - set dn_schema_region_consensus_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( - set dn_data_region_consensus_port=%%i - ) -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( - set dn_rpc_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( - set dn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_mpp_data_exchange_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_schema_region_consensus_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_data_region_consensus_port=%%i ) ) ELSE ( - echo "Can't find iotdb-datanode.properties, check the default ports" + echo "Can't find iotdb-system.properties or iotdb-datanode.properties, check the default ports" set dn_rpc_port=6667 set dn_internal_port=10730 set dn_mpp_data_exchange_port=10740 diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index 2197251570e5..d249d4275674 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -22,15 +22,21 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ +IF EXIST "%superior%\conf\iotdb-system.properties" ( + set config_file="%superior_dir%\conf\iotdb-system.properties" +) ELSE ( + set config_file="%superior_dir%\conf\iotdb-datanode.properties" +) + for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" -"%superior_dir%\conf\iotdb-datanode.properties"') do ( +"%config_file%"') do ( set dn_rpc_port=%%i ) echo Check whether the rpc_port is used..., port is %dn_rpc_port% for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" -"%superior_dir%\conf\iotdb-datanode.properties"') do ( +"%config_file%"') do ( set dn_rpc_address=%%i ) diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh index cb253f1a1edf..71bdcef0a82b 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh @@ -19,7 +19,12 @@ # DATANODE_CONF="`dirname "$0"`/../conf" -dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-datanode.properties` + +if [ -f "${DATANODE_CONF}/iotdb-system.properties" ]; then + dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-system.properties` +else + dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-datanode.properties` +fi force="" diff --git a/iotdb-core/datanode/src/assembly/server.xml b/iotdb-core/datanode/src/assembly/server.xml index b1a302597945..96bfdd92748a 100644 --- a/iotdb-core/datanode/src/assembly/server.xml +++ b/iotdb-core/datanode/src/assembly/server.xml @@ -39,8 +39,8 @@ - ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties - conf/iotdb-common.properties + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties + conf/iotdb-system.properties diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index a532cbe947c8..86ad6087fc08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -87,9 +88,9 @@ public class IoTDBDescriptor { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDescriptor.class); - private final CommonDescriptor commonDescriptor = CommonDescriptor.getInstance(); + private static final CommonDescriptor commonDescriptor = CommonDescriptor.getInstance(); - private final IoTDBConfig conf = new IoTDBConfig(); + private static final IoTDBConfig conf = new IoTDBConfig(); private static final long MAX_THROTTLE_THRESHOLD = 600 * 1024 * 1024 * 1024L; @@ -99,6 +100,19 @@ public class IoTDBDescriptor { private static final double MIN_DIR_USE_PROPORTION = 0.5; + static { + ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); + URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); + URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); + URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); + try { + updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + } catch (Exception e) { + LOGGER.error("Failed to update config file", e); + } + } + protected IoTDBDescriptor() { loadProps(); ServiceLoader propertiesLoaderServiceLoader = @@ -136,7 +150,7 @@ public IoTDBConfig getConfig() { * * @return url object if location exit, otherwise null. */ - public URL getPropsUrl(String configFileName) { + public static URL getPropsUrl(String configFileName) { String urlString = commonDescriptor.getConfDir(); if (urlString == null) { // If urlString wasn't provided, try to find a default config in the root of the classpath. @@ -174,28 +188,9 @@ else if (!urlString.endsWith(".properties")) { /** load a property file and set TsfileDBConfig variables. */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void loadProps() { - URL url = getPropsUrl(CommonConfig.CONFIG_NAME); Properties commonProperties = new Properties(); - if (url != null) { - try (InputStream inputStream = url.openStream()) { - LOGGER.info("Start to read config file {}", url); - commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } catch (FileNotFoundException e) { - LOGGER.error("Fail to find config file {}, reject DataNode startup.", url, e); - System.exit(-1); - } catch (IOException e) { - LOGGER.error("Cannot load config file, reject DataNode startup.", e); - System.exit(-1); - } catch (Exception e) { - LOGGER.error("Incorrect format in config file, reject DataNode startup.", e); - System.exit(-1); - } - } else { - LOGGER.warn( - "Couldn't load the configuration {} from any of the known sources.", - CommonConfig.CONFIG_NAME); - } - url = getPropsUrl(IoTDBConfig.CONFIG_NAME); + // if new properties file exist, skip old properties files + URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url != null) { try (InputStream inputStream = url.openStream()) { LOGGER.info("Start to read config file {}", url); @@ -216,7 +211,7 @@ private void loadProps() { // update all data seriesPath conf.updatePath(); commonDescriptor.getConfig().updatePath(System.getProperty(IoTDBConstant.IOTDB_HOME, null)); - MetricConfigDescriptor.getInstance().loadProps(commonProperties); + MetricConfigDescriptor.getInstance().loadProps(commonProperties, false); MetricConfigDescriptor.getInstance() .getMetricConfig() .updateRpcInstance( @@ -225,7 +220,7 @@ private void loadProps() { } else { LOGGER.warn( "Couldn't load the configuration {} from any of the known sources.", - IoTDBConfig.CONFIG_NAME); + CommonConfig.SYSTEM_CONFIG_NAME); } } @@ -1835,7 +1830,7 @@ public void loadHotModifiedProps(Properties properties) throws QueryProcessExcep } public void loadHotModifiedProps() throws QueryProcessException { - URL url = getPropsUrl(CommonConfig.CONFIG_NAME); + URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url == null) { LOGGER.warn("Couldn't load the configuration from any of the known sources."); return; @@ -1845,29 +1840,14 @@ public void loadHotModifiedProps() throws QueryProcessException { try (InputStream inputStream = url.openStream()) { LOGGER.info("Start to reload config file {}", url); commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } catch (Exception e) { - LOGGER.warn("Fail to reload config file {}", url, e); - throw new QueryProcessException( - String.format("Fail to reload config file %s because %s", url, e.getMessage())); - } - - url = getPropsUrl(IoTDBConfig.CONFIG_NAME); - if (url == null) { - LOGGER.warn("Couldn't load the configuration from any of the known sources."); - return; - } - try (InputStream inputStream = url.openStream()) { - LOGGER.info("Start to reload config file {}", url); - Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - commonProperties.putAll(properties); loadHotModifiedProps(commonProperties); } catch (Exception e) { LOGGER.warn("Fail to reload config file {}", url, e); throw new QueryProcessException( String.format("Fail to reload config file %s because %s", url, e.getMessage())); } - ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(commonProperties); + ReloadLevel reloadLevel = + MetricConfigDescriptor.getInstance().loadHotProps(commonProperties, false); LOGGER.info("Reload metric service in level {}", reloadLevel); if (reloadLevel == ReloadLevel.RESTART_INTERNAL_REPORTER) { IoTDBInternalReporter internalReporter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index 50c698079ff1..64c0f65fe303 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -20,8 +20,6 @@ package org.apache.iotdb.db.conf.rest; public class IoTDBRestServiceConfig { - static final String CONFIG_NAME = "iotdb-datanode.properties"; - static final String OLD_CONFIG_NAME = "iotdb-common.properties"; /** If the enableRestService is true, we will start REST Service. */ private boolean enableRestService = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index e0d0df3dee95..48725ef1839d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.conf.rest; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -40,11 +41,9 @@ public class IoTDBRestServiceDescriptor { private final IoTDBRestServiceConfig conf = new IoTDBRestServiceConfig(); protected IoTDBRestServiceDescriptor() { - Properties properties = loadProps(IoTDBRestServiceConfig.CONFIG_NAME); - if (properties != null) { - if (!properties.containsKey("enable_rest_service")) { - properties = loadProps(IoTDBRestServiceConfig.OLD_CONFIG_NAME); - } + URL systemConfig = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (systemConfig != null) { + Properties properties = loadProps(CommonConfig.SYSTEM_CONFIG_NAME); if (properties != null) { loadProps(properties); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java index 7a9a16aa20db..9cf9d51cf211 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java @@ -49,7 +49,7 @@ public class TagLogFile implements AutoCloseable { private FileChannel fileChannel; private static final String LENGTH_EXCEED_MSG = "Tag/Attribute exceeds the max length limit. " - + "Please enlarge tag_attribute_total_size in iotdb-common.properties"; + + "Please enlarge tag_attribute_total_size in iotdb-system.properties"; private static final int MAX_LENGTH = CommonDescriptor.getInstance().getConfig().getTagAttributeTotalSize(); diff --git a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties deleted file mode 100644 index 143c0e0b7f00..000000000000 --- a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -timestamp_precision=ms -udf_lib_dir=target/datanode1/ext/udf -trigger_lib_dir=target/datanode1/ext/trigger -pipe_lib_dir=target/datanode1/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-system.properties similarity index 88% rename from iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties rename to iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-system.properties index a7ed41f26152..5bccd18242b0 100644 --- a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties +++ b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-system.properties @@ -39,4 +39,9 @@ sort_tmp_dir=target/datanode1/tmp dn_metric_reporter_list=PROMETHEUS dn_metric_level=IMPORTANT dn_metric_async_collect_period=5 -dn_metric_prometheus_reporter_port=9097 \ No newline at end of file +dn_metric_prometheus_reporter_port=9097 + +timestamp_precision=ms +udf_lib_dir=target/datanode1/ext/udf +trigger_lib_dir=target/datanode1/ext/trigger +pipe_lib_dir=target/datanode1/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties deleted file mode 100644 index 9cf060d61fdd..000000000000 --- a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -timestamp_precision=ms -udf_lib_dir=target/datanode2/ext/udf -trigger_lib_dir=target/datanode2/ext/trigger -pipe_lib_dir=target/datanode2/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-system.properties similarity index 90% rename from iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties rename to iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-system.properties index a8508a79678d..26f56c3c03ce 100644 --- a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties +++ b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-system.properties @@ -40,3 +40,8 @@ dn_metric_reporter_list=PROMETHEUS dn_metric_level=IMPORTANT dn_metric_async_collect_period=5 dn_metric_prometheus_reporter_port=9099 + +timestamp_precision=ms +udf_lib_dir=target/datanode2/ext/udf +trigger_lib_dir=target/datanode2/ext/trigger +pipe_lib_dir=target/datanode2/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties deleted file mode 100644 index 1b48e21f3037..000000000000 --- a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -timestamp_precision=ms -udf_lib_dir=target/datanode3/ext/udf -trigger_lib_dir=target/datanode3/ext/trigger -pipe_lib_dir=target/datanode3/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-system.properties similarity index 88% rename from iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-datanode.properties rename to iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-system.properties index d4faf82e09b3..9bde59103778 100644 --- a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-datanode.properties +++ b/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-system.properties @@ -39,4 +39,9 @@ sort_tmp_dir=target/datanode3/tmp dn_metric_reporter_list=PROMETHEUS dn_metric_level=IMPORTANT dn_metric_async_collect_period=5 -dn_metric_prometheus_reporter_port=9101 \ No newline at end of file +dn_metric_prometheus_reporter_port=9101 + +timestamp_precision=ms +udf_lib_dir=target/datanode3/ext/udf +trigger_lib_dir=target/datanode3/ext/trigger +pipe_lib_dir=target/datanode3/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties new file mode 100644 index 000000000000..86943f5ed347 --- /dev/null +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +base_dir=target/tmp +dn_data_dirs=target/data +dn_wal_dirs=target/wal +index_root_dir=target/index +dn_tracing_dir=target/data/tracing +minimum_pbtree_segment_in_bytes=0 +page_cache_in_pbtree=10 +dn_internal_address=0.0.0.0 +dn_sync_dir=target/sync +dn_seed_config_node=127.0.0.1:10710 + + +#################### +### REST Service Configuration +#################### + +# Is the REST service enabled +enable_rest_service=true + +# the binding port of the REST service +# rest_service_port=18080 + +# Whether to display rest service interface information through swagger. eg: http://ip:port/swagger.json +# enable_swagger=false + +# the default row limit to a REST query response when the rowSize parameter is not given in request +# rest_query_default_row_size_limit=10000 + +# the expiration time of the user login information cache (in seconds) +# cache_expire_in_seconds=28800 + +# maximum number of users can be stored in the user login cache. +# cache_max_num=100 + +# init capacity of users can be stored in the user login cache. +# cache_init_num=10 + +# is SSL enabled +# enable_https=false + +# SSL key store path +# key_store_path= + +# SSL key store password +# key_store_pwd= + +# Is client authentication required +# client_auth=false + +# SSL trust store path +# trust_store_path= + +# SSL trust store password. +# trust_store_pwd= + +# SSL timeout (in seconds) +# idle_timeout_in_seconds=50000 diff --git a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java index e053587713da..f1ec288d47ad 100644 --- a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java +++ b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java @@ -37,8 +37,8 @@ private MetricConfigDescriptor() { } /** Load properties into metric config. */ - public void loadProps(Properties properties) { - MetricConfig loadConfig = generateFromProperties(properties); + public void loadProps(Properties properties, boolean isConfigNode) { + MetricConfig loadConfig = generateFromProperties(properties, isConfigNode); metricConfig.copy(loadConfig); } @@ -47,8 +47,8 @@ public void loadProps(Properties properties) { * * @return reload level of metric service */ - public ReloadLevel loadHotProps(Properties properties) { - MetricConfig newMetricConfig = generateFromProperties(properties); + public ReloadLevel loadHotProps(Properties properties, boolean isConfigNode) { + MetricConfig newMetricConfig = generateFromProperties(properties, isConfigNode); ReloadLevel reloadLevel = ReloadLevel.NOTHING; if (!metricConfig.equals(newMetricConfig)) { if (!metricConfig.getMetricLevel().equals(newMetricConfig.getMetricLevel()) @@ -72,7 +72,7 @@ public ReloadLevel loadHotProps(Properties properties) { } /** Load properties into metric config. */ - private MetricConfig generateFromProperties(Properties properties) { + private MetricConfig generateFromProperties(Properties properties, boolean isConfigNode) { MetricConfig loadConfig = new MetricConfig(); String reporterList = @@ -83,73 +83,98 @@ private MetricConfig generateFromProperties(Properties properties) { loadConfig.getMetricReporterList().stream() .map(ReporterType::toString) .collect(Collectors.toSet())), - properties); + properties, + isConfigNode); loadConfig.setMetricReporterList(reporterList); loadConfig.setMetricLevel( MetricLevel.valueOf( - getProperty("metric_level", String.valueOf(loadConfig.getMetricLevel()), properties))); + getProperty( + "metric_level", + String.valueOf(loadConfig.getMetricLevel()), + properties, + isConfigNode))); loadConfig.setAsyncCollectPeriodInSecond( Integer.parseInt( getProperty( "metric_async_collect_period", String.valueOf(loadConfig.getAsyncCollectPeriodInSecond()), - properties))); + properties, + isConfigNode))); loadConfig.setPrometheusReporterPort( Integer.parseInt( getProperty( "metric_prometheus_reporter_port", String.valueOf(loadConfig.getPrometheusReporterPort()), - properties))); + properties, + isConfigNode))); IoTDBReporterConfig reporterConfig = loadConfig.getIoTDBReporterConfig(); reporterConfig.setHost( - getProperty("metric_iotdb_reporter_host", reporterConfig.getHost(), properties)); + getProperty( + "metric_iotdb_reporter_host", reporterConfig.getHost(), properties, isConfigNode)); reporterConfig.setPort( Integer.valueOf( getProperty( "metric_iotdb_reporter_port", String.valueOf(reporterConfig.getPort()), - properties))); + properties, + isConfigNode))); reporterConfig.setUsername( - getProperty("metric_iotdb_reporter_username", reporterConfig.getUsername(), properties)); + getProperty( + "metric_iotdb_reporter_username", + reporterConfig.getUsername(), + properties, + isConfigNode)); reporterConfig.setPassword( - getProperty("metric_iotdb_reporter_password", reporterConfig.getPassword(), properties)); + getProperty( + "metric_iotdb_reporter_password", + reporterConfig.getPassword(), + properties, + isConfigNode)); reporterConfig.setMaxConnectionNumber( Integer.valueOf( getProperty( "metric_iotdb_reporter_max_connection_number", String.valueOf(reporterConfig.getMaxConnectionNumber()), - properties))); + properties, + isConfigNode))); reporterConfig.setLocation( - getProperty("metric_iotdb_reporter_location", reporterConfig.getLocation(), properties)); + getProperty( + "metric_iotdb_reporter_location", + reporterConfig.getLocation(), + properties, + isConfigNode)); reporterConfig.setPushPeriodInSecond( Integer.valueOf( getProperty( "metric_iotdb_reporter_push_period", String.valueOf(reporterConfig.getPushPeriodInSecond()), - properties))); - loadConfig.setInternalReportType( - InternalReporterType.valueOf( - properties.getProperty( - "dn_metric_internal_reporter_type", - loadConfig.getInternalReportType().toString()))); + properties, + isConfigNode))); + if (!isConfigNode) { + loadConfig.setInternalReportType( + InternalReporterType.valueOf( + properties.getProperty( + "dn_metric_internal_reporter_type", + loadConfig.getInternalReportType().toString()))); + } return loadConfig; } /** Get property from confignode or datanode. */ - private String getProperty(String target, String defaultValue, Properties properties) { - return properties.getProperty( - "dn_" + target, properties.getProperty("cn_" + target, defaultValue)); + private String getProperty( + String target, String defaultValue, Properties properties, boolean isConfigNode) { + return properties.getProperty((isConfigNode ? "cn_" : "dn_") + target, defaultValue); } private static class MetricConfigDescriptorHolder { diff --git a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java index 3ce4b79c6b78..43382d00aedd 100644 --- a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java +++ b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java @@ -47,7 +47,7 @@ public void testConfigNodeMetricConfig() { properties.setProperty("cn_metric_iotdb_reporter_location", "metric"); properties.setProperty("cn_metric_iotdb_reporter_push_period", "5"); - MetricConfigDescriptor.getInstance().loadProps(properties); + MetricConfigDescriptor.getInstance().loadProps(properties, true); MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); @@ -84,7 +84,7 @@ public void testDataNodeMetricConfig() { properties.setProperty("dn_metric_iotdb_reporter_push_period", "5"); properties.setProperty("dn_metric_internal_reporter_type", "IOTDB"); - MetricConfigDescriptor.getInstance().loadProps(properties); + MetricConfigDescriptor.getInstance().loadProps(properties, false); MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties similarity index 71% rename from iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties rename to iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties index 27443e02918b..840e787c5b79 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -15,6 +15,439 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# + +#################### +### Config Node RPC Configuration +#################### + +# Used for RPC communication inside cluster. +# Could set 127.0.0.1(for local test) or ipv4 address. +# Datatype: String +cn_internal_address=127.0.0.1 + +# Used for RPC communication inside cluster. +# Datatype: int +cn_internal_port=10710 + +# Used for consensus communication among ConfigNodes inside cluster. +# Datatype: int +cn_consensus_port=10720 + +#################### +### Seed ConfigNode +#################### + +# For the first ConfigNode to start, cn_seed_config_node points to its own cn_internal_address:cn_internal_port. +# For other ConfigNodes that to join the cluster, cn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. +# Note: After this ConfigNode successfully joins the cluster for the first time, this parameter is no longer used. +# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. +# Format: address:port e.g. 127.0.0.1:10710 +# Datatype: String +cn_seed_config_node=127.0.0.1:10710 + +#################### +### Directory configuration +#################### + +# system dir +# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/system). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# cn_system_dir=data\\confignode\\system +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# cn_system_dir=data/confignode/system + +# consensus dir +# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/consensus). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. +# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# cn_consensus_dir=data\\confignode\\consensus +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# cn_consensus_dir=data/confignode/consensus + +# pipe_receiver_file_dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${cn_system_dir}/pipe/receiver). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If pipe_receiver_file_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dir=data\\confignode\\system\\pipe\\receiver +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dir=data/confignode/system/pipe/receiver + +#################### +### thrift rpc configuration +#################### + +# this feature is under development, set this as false before it is done. +# Datatype: boolean +# cn_rpc_thrift_compression_enable=false + +# if true, a snappy based compression method will be called before sending data by the network +# Datatype: boolean +# this feature is under development, set this as false before it is done. +# cn_rpc_advanced_compression_enable=false + +# Datatype: int +# cn_rpc_max_concurrent_client_num=65535 + +# thrift max frame size, 512MB by default +# Datatype: int +# cn_thrift_max_frame_size=536870912 + +# thrift init buffer size +# Datatype: int +# cn_thrift_init_buffer_size=1024 + +# Thrift socket and connection timeout between raft nodes, in milliseconds. +# Datatype: int +# cn_connection_timeout_ms=60000 + +# selector thread (TAsyncClientManager) nums for async thread in a clientManager +# Datatype: int +# cn_selector_thread_nums_of_client_manager=1 + +# The maximum number of clients that can be allocated for a node in a clientManager. +# when the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked +# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. +# Datatype: int +# cn_max_client_count_for_each_node_in_client_manager=300 + +#################### +### Metric Configuration +#################### + +# The reporters of metric module to report metrics +# If there are more than one reporter, please separate them by commas ",". +# Options: [JMX, PROMETHEUS] +# Datatype: String +# cn_metric_reporter_list= + +# The level of metric module +# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] +# Datatype: String +# cn_metric_level=CORE + +# The period of async collection of some metrics in second +# Datatype: int +# cn_metric_async_collect_period=5 + +# The port of prometheus reporter of metric module +# Datatype: int +# cn_metric_prometheus_reporter_port=9091 + + + +#################### +### Data Node RPC Configuration +#################### + +# Used for connection of IoTDB native clients(Session) +# Could set 127.0.0.1(for local test) or ipv4 address +# Datatype: String +dn_rpc_address=0.0.0.0 + +# Used for connection of IoTDB native clients(Session) +# Bind with dn_rpc_address +# Datatype: int +dn_rpc_port=6667 + +# Used for communication inside cluster. +# could set 127.0.0.1(for local test) or ipv4 address. +# Datatype: String +dn_internal_address=127.0.0.1 + +# Used for communication inside cluster. +# Bind with dn_internal_address +# Datatype: int +dn_internal_port=10730 + +# Port for data exchange among DataNodes inside cluster +# Bind with dn_internal_address +# Datatype: int +dn_mpp_data_exchange_port=10740 + +# port for consensus's communication for schema region inside cluster. +# Bind with dn_internal_address +# Datatype: int +dn_schema_region_consensus_port=10750 + +# port for consensus's communication for data region inside cluster. +# Bind with dn_internal_address +# Datatype: int +dn_data_region_consensus_port=10760 + +# Datatype: long +# The time of data node waiting for the next retry to join into the cluster. +# dn_join_cluster_retry_interval_ms=5000 + +#################### +### SSL Configuration +#################### + +# Does dn_rpc_port enable SSL +# enable_thrift_ssl=false + +# Rest Service enabled SSL +# enable_https=false + +# SSL key store path +# linux e.g. /home/iotdb/server.keystore (absolute path) or server.keystore (relative path) +# windows e.g. C:\\iotdb\\server.keystore (absolute path) or server.keystore (relative path) +# key_store_path= + +# SSL key store password +# key_store_pwd= + +#################### +### Seed ConfigNode +#################### + +# dn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. +# Note: After this DataNode successfully joins the cluster for the first time, this parameter is no longer used. +# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. +# Format: address:port e.g. 127.0.0.1:10710 +# Datatype: String +dn_seed_config_node=127.0.0.1:10710 + +#################### +### Connection Configuration +#################### + +# The maximum session idle time. unit: ms +# Idle sessions are the ones that performs neither query or non-query operations for a period of time +# Set to 0 to disable session timeout +# Datatype: int +# dn_session_timeout_threshold=0 + +# Datatype: boolean +# dn_rpc_thrift_compression_enable=false + +# if true, a snappy based compression method will be called before sending data by the network +# Datatype: boolean +# this feature is under development, set this as false before it is done. +# dn_rpc_advanced_compression_enable=false + +# Datatype: int +# dn_rpc_selector_thread_count=1 + +# Datatype: int +# dn_rpc_min_concurrent_client_num=1 + +# Datatype: int +# dn_rpc_max_concurrent_client_num=65535 + +# thrift max frame size, 512MB by default +# Datatype: int +# dn_thrift_max_frame_size=536870912 + +# thrift init buffer size +# Datatype: int +# dn_thrift_init_buffer_size=1024 + +# Thrift socket and connection timeout between raft nodes, in milliseconds. +# Datatype: int +# dn_connection_timeout_ms=60000 + +# selector thread (TAsyncClientManager) nums for async thread in a clientManager +# Datatype: int +# dn_selector_thread_count_of_client_manager=1 + +# The maximum number of clients that can be allocated for a node in a clientManager. +# When the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked +# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. +# Datatype: int +# dn_max_client_count_for_each_node_in_client_manager=300 + +#################### +### Directory Configuration +#################### + +# system dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/system). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_system_dir=data\\datanode\\system +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_system_dir=data/datanode/system + + +# data dirs +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/data). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# If there are more than one directory, please separate them by commas ",". +# Note: If data_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_data_dirs=data\\datanode\\data +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_data_dirs=data/datanode/data + + +# multi_dir_strategy +# The strategy is used to choose a directory from data_dirs for the system to store a new tsfile. +# System provides two strategies to choose from, or user can create his own strategy by extending org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy. +# The info of the two strategies are as follows: +# 1. SequenceStrategy: the system will choose the directory in sequence. +# 2. MaxDiskUsableSpaceFirstStrategy: the system will choose the directory whose disk has the maximum space. +# Set SequenceStrategy or MaxDiskUsableSpaceFirstStrategy to apply the corresponding strategy. +# If this property is unset, system will use SequenceStrategy as default strategy. +# For this property, fully-qualified class name (include package name) and simple class name are both acceptable. +# dn_multi_dir_strategy=SequenceStrategy + +# consensus dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If consensus_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_consensus_dir=data\\datanode\\consensus +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_consensus_dir=data/datanode/consensus + +# wal dirs +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# If there are more than one directory, please separate them by commas ",". +# Note: If wal_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_wal_dirs=data\\datanode\\wal +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_wal_dirs=data/datanode/wal + +# tracing dir +# Uncomment following fields to configure the tracing root directory. +# For Windows platform, the index is as follows: +# dn_tracing_dir=datanode\\tracing +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_tracing_dir=datanode/tracing + +# sync dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_sync_dir=data\\datanode\\sync +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_sync_dir=data/datanode/sync + +# sort_tmp_dir +# This property is used to configure the temporary directory for sorting operation. +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If sort_tmp_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# sort_tmp_dir=data\\datanode\\tmp +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# sort_tmp_dir=data/datanode/tmp + +# pipe_receiver_file_dirs +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${dn_system_dir}/pipe/receiver). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# If there are more than one directory, please separate them by commas ",". +# Note: If pipe_receiver_file_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dirs=data\\datanode\\system\\pipe\\receiver +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dirs=data/datanode/system/pipe/receiver + +#################### +### Metric Configuration +#################### + +# The reporters of metric module to report metrics +# If there are more than one reporter, please separate them by commas ",". +# Options: [JMX, PROMETHEUS] +# Datatype: String +# dn_metric_reporter_list= + +# The level of metric module +# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] +# Datatype: String +# dn_metric_level=CORE + +# The period of async collection of some metrics in second +# Datatype: int +# dn_metric_async_collect_period=5 + +# The port of prometheus reporter of metric module +# Datatype: int +dn_metric_prometheus_reporter_port=9092 + +# The type of internal reporter in metric module, used for checking flushed point number +# Options: [MEMORY, IOTDB] +# Datatype: String +# dn_metric_internal_reporter_type=MEMORY + +#################### +### REST Service Configuration +#################### + +# Is the REST service enabled +# enable_rest_service=false + +# the binding port of the REST service +# rest_service_port=18080 + +# Whether to display rest service interface information through swagger. eg: http://ip:port/swagger.json +# enable_swagger=false + +# the default row limit to a REST query response when the rowSize parameter is not given in request +# rest_query_default_row_size_limit=10000 + +# the expiration time of the user login information cache (in seconds) +# cache_expire_in_seconds=28800 + +# maximum number of users can be stored in the user login cache. +# cache_max_num=100 + +# init capacity of users can be stored in the user login cache. +# cache_init_num=10 + +# Is client authentication required +# client_auth=false + +# SSL trust store path +# trust_store_path="" + +# SSL trust store password. +# trust_store_pwd="" + +# SSL timeout (in seconds) +# idle_timeout_in_seconds=50000 + + + #################### ### Cluster Configuration diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat index 36691cff4095..0fd3bd5d9947 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat @@ -35,7 +35,11 @@ if not "%CLEAN_SERVICE%"=="y" if not "%CLEAN_SERVICE%"=="Y" ( start cmd /c "%IOTDB_HOME%\\sbin\\stop-confignode.bat -f" timeout /t 3 > nul rmdir /s /q "%IOTDB_HOME%\data\confignode\" 2>nul -set IOTDB_CONFIGNODE_CONFIG="%IOTDB_HOME%\conf\iotdb-confignode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set IOTDB_CONFIGNODE_CONFIG="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE ( + set IOTDB_CONFIGNODE_CONFIG="%IOTDB_HOME%\conf\iotdb-confignode.properties" +) set "delimiter=,;" for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_system_dir" %IOTDB_CONFIGNODE_CONFIG%') do ( diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh index 88a5e3488308..fee896993620 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh @@ -33,7 +33,12 @@ fi nohup bash ${IOTDB_HOME}/sbin/stop-confignode.sh -f >/dev/null 2>&1 & rm -rf ${IOTDB_HOME}/data/confignode/ >/dev/null 2>&1 & -IOTDB_CONFIGNODE_CONFIG=${IOTDB_HOME}/conf/iotdb-confignode.properties +if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + IOTDB_CONFIGNODE_CONFIG="${IOTDB_HOME}/conf/iotdb-system.properties" +else + IOTDB_CONFIGNODE_CONFIG="${IOTDB_HOME}/conf/iotdb-confignode.properties" +fi + cn_system_dir=$(echo $(grep '^cn_system_dir=' ${IOTDB_CONFIGNODE_CONFIG} || echo "data/confignode/system") | sed 's/.*=//') cn_consensus_dir=$(echo $(grep '^cn_consensus_dir=' ${IOTDB_CONFIGNODE_CONFIG} || echo "data/confignode/consensus") | sed 's/.*=//') diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat index 1ccf0346f504..26aab580e851 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat @@ -34,7 +34,11 @@ if not "%CLEAN_SERVICE%"=="y" if not "%CLEAN_SERVICE%"=="Y" ( start cmd /c "%IOTDB_HOME%\\sbin\\stop-datanode.bat -f" timeout /t 3 > nul rmdir /s /q "%IOTDB_HOME%\data\datanode\" 2>nul -set IOTDB_DATANODE_CONFIG="%IOTDB_HOME%\conf\iotdb-datanode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set IOTDB_DATANODE_CONFIG="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE ( + set IOTDB_DATANODE_CONFIG="%IOTDB_HOME%\conf\iotdb-datanode.properties" +) set "delimiter=,;" for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_system_dir" %IOTDB_DATANODE_CONFIG%') do ( diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh index 98e287ef4e6f..9a1580a2a9e6 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh @@ -32,7 +32,13 @@ fi nohup bash ${IOTDB_HOME}/sbin/stop-datanode.sh -f >/dev/null 2>&1 & rm -rf ${IOTDB_HOME}/data/datanode/ >/dev/null 2>&1 & -IOTDB_DATANODE_CONFIG=${IOTDB_HOME}/conf/iotdb-datanode.properties + +if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + IOTDB_DATANODE_CONFIG="${IOTDB_HOME}/conf/iotdb-system.properties" +else + IOTDB_DATANODE_CONFIG="${IOTDB_HOME}/conf/iotdb-datanode.properties" +fi + dn_system_dir=$(echo $(grep '^dn_system_dir=' ${IOTDB_DATANODE_CONFIG} || echo "data/datanode/system") | sed 's/.*=//') dn_data_dirs=$(echo $(grep '^dn_data_dirs=' ${IOTDB_DATANODE_CONFIG} || echo "data/datanode/data") | sed 's/.*=//') dn_consensus_dir=$(echo $(grep '^dn_consensus_dir=' ${IOTDB_DATANODE_CONFIG} || echo "data/datanode/consensus") | sed 's/.*=//') diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat index bb07c530befb..14a0e4700cbd 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat @@ -232,7 +232,11 @@ exit /b :local_dirs_check setlocal enabledelayedexpansion -set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +) for /f "usebackq tokens=1,* delims==" %%a in ("%properties_file%") do ( if "%%a"=="dn_data_dirs" ( set "dn_data_dirs=%%b" @@ -350,46 +354,36 @@ if defined operation_dirs ( exit /b :local_ports_check -IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-datanode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-datanode.properties" +) ELSE ( + set DATANODE_CONFIG_FILE_PATH= +) +IF DEFINED DATANODE_CONFIG_FILE_PATH ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_rpc_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_mpp_data_exchange_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_schema_region_consensus_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( - set dn_data_region_consensus_port=%%i - ) -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_rpc_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_mpp_data_exchange_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_schema_region_consensus_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_data_region_consensus_port=%%i ) ) ELSE ( @@ -400,24 +394,26 @@ IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( set dn_data_region_consensus_port=10760 ) -IF EXIST "%IOTDB_CONF%\iotdb-confignode.properties" ( +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-confignode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-confignode.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-confignode.properties" +) ELSE ( + set CONFIGNODE_CONFIG_FILE_PATH= +) +IF DEFINED CONFIGNODE_CONFIG_FILE_PATH ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - %IOTDB_CONF%\iotdb-confignode.properties') do ( + %CONFIGNODE_CONFIG_FILE_PATH%') do ( set cn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - %IOTDB_CONF%\iotdb-confignode.properties') do ( + %CONFIGNODE_CONFIG_FILE_PATH%') do ( set cn_consensus_port=%%i ) -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-confignode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - %IOTDB_HOME%\conf\iotdb-confignode.properties') do ( - set cn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - %IOTDB_HOME%\conf\iotdb-confignode.properties') do ( - set cn_consensus_port=%%i - ) ) ELSE ( set cn_internal_port=10710 set cn_consensus_port=10720 diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh index 997f7586da46..9cb08bb5e334 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh @@ -170,7 +170,12 @@ get_properties_value() { local file_name=$1 local property_name=$2 local default_value=$3 - local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties") + if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + local file_path="${IOTDB_HOME}/conf/iotdb-system.properties" + else + local file_path="${IOTDB_HOME}/conf/${file_name}.properties" + fi + local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${file_path}") if [ -z "$property_value" ]; then property_value="$default_value" fi diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh index 7539ec883489..753d1bccb051 100755 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh @@ -156,7 +156,19 @@ checkDataNodePortUsages () { echo "Warning: If you do not use sudo, the checking may not detect all the occupied ports." fi occupied=false - if [ -f "$IOTDB_CONF/iotdb-datanode.properties" ]; then + if [ -f "$IOTDB_CONF/iotdb-system.properties" ]; then + dn_rpc_port=$(sed '/^dn_rpc_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_internal_port=$(sed '/^dn_internal_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_mpp_data_exchange_port=$(sed '/^dn_mpp_data_exchange_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_schema_region_consensus_port=$(sed '/^dn_schema_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_data_region_consensus_port=$(sed '/^dn_data_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + elif [ -f "$IOTDB_HOME/conf/iotdb-system.properties" ]; then + dn_rpc_port=$(sed '/^dn_rpc_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_internal_port=$(sed '/^dn_internal_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_mpp_data_exchange_port=$(sed '/^dn_mpp_data_exchange_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_schema_region_consensus_port=$(sed '/^dn_schema_region_consensus_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_data_region_consensus_port=$(sed '/^dn_data_region_consensus_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + elif [ -f "$IOTDB_CONF/iotdb-datanode.properties" ]; then dn_rpc_port=$(sed '/^dn_rpc_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) dn_internal_port=$(sed '/^dn_internal_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) dn_mpp_data_exchange_port=$(sed '/^dn_mpp_data_exchange_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) @@ -169,7 +181,7 @@ checkDataNodePortUsages () { dn_schema_region_consensus_port=$(sed '/^dn_schema_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) dn_data_region_consensus_port=$(sed '/^dn_data_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) else - echo "Warning: cannot find iotdb-datanode.properties, check the default configuration" + echo "Warning: cannot find iotdb-system.properties or iotdb-datanode.properties, check the default configuration" fi dn_rpc_port=${dn_rpc_port:-6667} dn_internal_port=${dn_internal_port:-10730} @@ -244,14 +256,20 @@ checkConfigNodePortUsages () { echo "Warning: If you do not use sudo, the checking may not detect all the occupied ports." fi occupied=false - if [ -f "$CONFIGNODE_CONF/iotdb-confignode.properties" ]; then + if [ -f "$CONFIGNODE_CONF/iotdb-system.properties" ]; then + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) + cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) + elif [ -f "$CONFIGNODE_HOME/conf/iotdb-system.properties" ]; then + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-system.properties) + cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-system.properties) + elif [ -f "$CONFIGNODE_CONF/iotdb-confignode.properties" ]; then cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) elif [ -f "$CONFIGNODE_HOME/conf/iotdb-confignode.properties" ]; then cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-confignode.properties) cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-confignode.properties) else - echo "Cannot find iotdb-confignode.properties, check the default configuration" + echo "Cannot find iotdb-system.properties or iotdb-confignode.properties, check the default configuration" fi cn_internal_port=${cn_internal_port:-10710} cn_consensus_port=${cn_consensus_port:-10720} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 6e8c94340067..59de17e371ea 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -36,7 +36,10 @@ public class CommonConfig { - public static final String CONFIG_NAME = "iotdb-common.properties"; + public static final String OLD_CONFIG_NODE_CONFIG_NAME = "iotdb-confignode.properties"; + public static final String OLD_DATA_NODE_CONFIG_NAME = "iotdb-datanode.properties"; + public static final String OLD_COMMON_CONFIG_NAME = "iotdb-common.properties"; + public static final String SYSTEM_CONFIG_NAME = "iotdb-system.properties"; private static final Logger logger = LoggerFactory.getLogger(CommonConfig.class); // Open ID Secret diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java new file mode 100644 index 000000000000..49b44f0bc02f --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.conf; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URL; +import java.nio.file.Files; +import java.util.concurrent.TimeUnit; + +public class ConfigFileAutoUpdateTool { + + private final String lockFileSuffix = ".lock"; + private final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); + private final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); + private Logger logger = LoggerFactory.getLogger(ConfigFileAutoUpdateTool.class); + private String license = + "#\n" + + "# Licensed to the Apache Software Foundation (ASF) under one\n" + + "# or more contributor license agreements. See the NOTICE file\n" + + "# distributed with this work for additional information\n" + + "# regarding copyright ownership. The ASF licenses this file\n" + + "# to you under the Apache License, Version 2.0 (the\n" + + "# \"License\"); you may not use this file except in compliance\n" + + "# with the License. You may obtain a copy of the License at\n" + + "#\n" + + "# http://www.apache.org/licenses/LICENSE-2.0\n" + + "#\n" + + "# Unless required by applicable law or agreed to in writing,\n" + + "# software distributed under the License is distributed on an\n" + + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" + + "# KIND, either express or implied. See the License for the\n" + + "# specific language governing permissions and limitations\n" + + "# under the License."; + + public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) + throws IOException, InterruptedException { + if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { + return; + } + File systemFile = new File(systemUrl.getFile()); + File configNodeFile = new File(configNodeUrl.getFile()); + File dataNodeFile = new File(dataNodeUrl.getFile()); + File commonFile = new File(commonUrl.getFile()); + + if (systemFile.exists()) { + return; + } + boolean canUpdate = (configNodeFile.exists() || dataNodeFile.exists()) && commonFile.exists(); + if (!canUpdate) { + return; + } + + File lockFile = new File(systemFile.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + try { + // other progress updated this file + if (systemFile.exists()) { + return; + } + try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { + raf.write(license.getBytes()); + String configNodeContent = readConfigLines(configNodeFile); + raf.write(configNodeContent.getBytes()); + String dataNodeContent = readConfigLines(dataNodeFile); + raf.write(dataNodeContent.getBytes()); + String commonContent = readConfigLines(commonFile); + raf.write(commonContent.getBytes()); + } + Files.move(lockFile.toPath(), systemFile.toPath()); + } finally { + releaseFileLock(lockFile); + } + } + + private String readConfigLines(File file) throws IOException { + if (!file.exists()) { + return ""; + } + byte[] bytes = Files.readAllBytes(file.toPath()); + String content = new String(bytes); + return content.replace(license, ""); + } + + private void acquireTargetFileLock(File file) throws IOException, InterruptedException { + long totalWaitTime = 0; + while (totalWaitTime < maxTimeMillsToAcquireLock) { + if (file.createNewFile()) { + return; + } + totalWaitTime += waitTimeMillsPerCheck; + Thread.sleep(waitTimeMillsPerCheck); + } + logger.warn( + "Waiting for {} seconds to acquire configuration file update lock." + + " There may have been an unexpected interruption in the last" + + " configuration file update. Ignore temporary file {}", + totalWaitTime / 1000, + file.getName()); + } + + private void releaseFileLock(File file) throws IOException { + Files.deleteIfExists(file.toPath()); + } +}