Skip to content

Commit

Permalink
Merge config files to iotdb-system.properties (apache#12570)
Browse files Browse the repository at this point in the history
Co-authored-by: Haonan <hhaonan@outlook.com>
  • Loading branch information
2 people authored and SzyWilliam committed Nov 22, 2024
1 parent e74e233 commit d921bdb
Show file tree
Hide file tree
Showing 57 changed files with 1,022 additions and 924 deletions.
5 changes: 4 additions & 1 deletion .github/workflows/multi-language-client.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ jobs:
- name: Integration test
shell: bash
run: |
cd iotdb-client/client-go/ && make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo
cd iotdb-client
git clone https://github.com/apache/iotdb-client-go.git
cd iotdb-client-go
make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo
python:
runs-on: ubuntu-latest
Expand Down
2 changes: 1 addition & 1 deletion docker/src/main/DockerCompose/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ function on_stop(){
trap 'on_stop' SIGTERM SIGKILL SIGQUIT


replace-conf-from-env.sh ${start_what}
replace-conf-from-env.sh

case "$1" in
datanode)
Expand Down
13 changes: 1 addition & 12 deletions docker/src/main/DockerCompose/replace-conf-from-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#

conf_path=${IOTDB_HOME}/conf
target_files="iotdb-system.properties"

function process_single(){
local key_value="$1"
Expand Down Expand Up @@ -48,17 +49,5 @@ function replace_configs(){
done
}

case "$1" in
confignode)
target_files="iotdb-common.properties iotdb-confignode.properties"
;;
datanode)
target_files="iotdb-common.properties iotdb-datanode.properties"
;;
all)
target_files="iotdb-common.properties iotdb-confignode.properties iotdb-datanode.properties"
;;
esac

replace_configs

14 changes: 7 additions & 7 deletions docker/src/main/Dockerfile-1c1d
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,13 @@ RUN apt update \
&& rm /apache-iotdb-*-bin.zip \
&& mv /apache-iotdb-* /iotdb \
&& mv /start-1c1d.sh /iotdb/sbin \
&& sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-datanode.properties \
&& sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-confignode.properties \
&& sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-confignode.properties \
&& sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-datanode.properties \
&& sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \
&& sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \
&& sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \
&& sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \
&& sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \
&& sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \
&& sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \
&& sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \
&& sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \
&& sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \
&& apt remove unzip -y \
&& apt autoremove -y \
&& apt purge --auto-remove -y \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ public class ClusterConstant {
public static final String CONFIG_NODE_PROPERTIES_FILE = "iotdb-confignode.properties";
public static final String DATA_NODE_PROPERTIES_FILE = "iotdb-datanode.properties";
public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties";
public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties";

public static final String SYSTEM_PROPERTIES_FILE = "system.properties";
public static final String CONFIG_NODE_SYSTEM_PROPERTIES_FILE = "confignode-system.properties";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public final void updateProperties(@Nullable String filePath) throws IOException
* @throws IOException if properties storage failed.
*/
public final void persistent(String filePath) throws IOException {
try (FileWriter confOutput = new FileWriter(filePath)) {
try (FileWriter confOutput = new FileWriter(filePath, true)) {
properties.store(confOutput, null);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -301,8 +301,8 @@ public final void changeConfig(
outputNodeConfig.updateProperties(immutableNodeProperties);

// Persistent
outputCommonConfig.persistent(getTargetCommonConfigPath());
outputNodeConfig.persistent(getTargetNodeConfigPath());
outputCommonConfig.persistent(getSystemConfigPath());
outputNodeConfig.persistent(getSystemConfigPath());
} catch (IOException ex) {
throw new AssertionError("Change the config of node failed. " + ex);
}
Expand Down Expand Up @@ -662,9 +662,7 @@ private String getKillPoints() {

protected abstract void renameFile();

protected abstract String getTargetNodeConfigPath();

protected abstract String getTargetCommonConfigPath();
protected abstract String getSystemConfigPath();

/** Return the node config file path specified through system variable */
protected abstract String getDefaultNodeConfigPath();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,16 @@
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_CONSENSUS_DIR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_METRIC_IOTDB_REPORTER_HOST;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_SYSTEM_DIR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_SYSTEM_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_PROPERTIES;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET;
Expand Down Expand Up @@ -90,13 +89,8 @@ public ConfigNodeWrapper(
}

@Override
protected String getTargetNodeConfigPath() {
return workDirFilePath("conf", CONFIG_NODE_PROPERTIES_FILE);
}

@Override
protected String getTargetCommonConfigPath() {
return workDirFilePath("conf", COMMON_PROPERTIES_FILE);
protected String getSystemConfigPath() {
return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,11 @@
import java.util.List;

import static org.apache.iotdb.consensus.ConsensusFactory.SIMPLE_CONSENSUS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_INIT_HEAP_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_DIRECT_MEMORY_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES;
Expand All @@ -53,6 +51,7 @@
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_SYSTEM_DIR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_TRACING_DIR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_WAL_DIRS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAIN_CLASS_NAME;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAX_TSBLOCK_SIZE_IN_BYTES;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_HOST;
Expand Down Expand Up @@ -124,13 +123,8 @@ public DataNodeWrapper(
}

@Override
protected String getTargetNodeConfigPath() {
return workDirFilePath("conf", DATA_NODE_PROPERTIES_FILE);
}

@Override
protected String getTargetCommonConfigPath() {
return workDirFilePath("conf", COMMON_PROPERTIES_FILE);
protected String getSystemConfigPath() {
return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE);
}

@Override
Expand Down
7 changes: 6 additions & 1 deletion iotdb-client/cli/src/assembly/resources/tools/backup.bat
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@ IF EXIST "%IOTDB_CONF%\datanode-env.bat" (
echo Can't find datanode-env.bat
)

IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" (
IF EXIST "%IOTDB_CONF%\iotdb-system.properties" (
for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port"
"%IOTDB_CONF%\iotdb-system.properties"') do (
set dn_rpc_port=%%i
)
) ELSE IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" (
for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port"
"%IOTDB_CONF%\iotdb-datanode.properties"') do (
set dn_rpc_port=%%i
Expand Down
7 changes: 6 additions & 1 deletion iotdb-client/cli/src/assembly/resources/tools/backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,12 @@ get_properties_value() {
local file_name=$1
local property_name=$2
local default_value=$3
local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties")
if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then
local file_path="${IOTDB_HOME}/conf/iotdb-system.properties"
else
local file_path="${IOTDB_HOME}/conf/${file_name}.properties"
fi
local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${file_path}")
if [ -z "$property_value" ]; then
property_value="$default_value"
fi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ set "passwd_param=root"
set "host_param=127.0.0.1"
set "port_param=6667"

set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties"
if exist "%IOTDB_HOME%\conf\iotdb-system.properties" (
set "properties_file=%IOTDB_HOME%\conf\iotdb-system.properties"
) else (
set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties"
)
set "key=dn_data_dirs"

for /f "usebackq tokens=1,* delims==" %%a in ("%properties_file%") do (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,11 @@ choose_unit() {
echo "$unit"
}

properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties"
if [ -f "$IOTDB_HOME/conf/iotdb-system.properties" ]; then
properties_file="$IOTDB_HOME/conf/iotdb-system.properties"
else
properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties"
fi
data_dir_key="dn_data_dirs"
value=$(get_property_value "$properties_file" "$data_dir_key")
if [ -n "$value" ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

package org.apache.iotdb.tool;

import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;

import org.slf4j.Logger;
Expand Down Expand Up @@ -62,8 +62,6 @@ public class IoTDBDataBackTool {
static String targetWalDirParam = "";
static String remoteDnDataDir = "";

static String DATA_NODE_CONF_NAME = IoTDBConfig.CONFIG_NAME;
static String CONFIG_NODE_CONF_NAME = "iotdb-confignode.properties";
static AtomicInteger fileCount = new AtomicInteger(0);
static AtomicInteger targetFileCount = new AtomicInteger(0);
static AtomicInteger processFileCount = new AtomicInteger(0);
Expand Down Expand Up @@ -177,10 +175,10 @@ public static void main(String[] args) {
System.setProperty("IOTDB_HOME", System.getenv("IOTDB_HOME"));
argsParse(args);
File sourceDir = new File(sourcePath);
Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME);
initDataNodeProperties(dataProperties);
Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME);
initConfigNodeProperties(configProperties);

Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
initDataNodeProperties(properties);
initConfigNodeProperties(properties);

StringBuilder targetDirString = new StringBuilder();
Map<String, String> copyMap = new HashMap<>();
Expand Down Expand Up @@ -215,7 +213,7 @@ public static void main(String[] args) {
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ CONFIG_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
cnMapProperties);
} else if (nodeTypeParam.equalsIgnoreCase("datanode")) {
countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties);
Expand All @@ -237,7 +235,7 @@ public static void main(String[] args) {
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ DATA_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
dnMapProperties);
} else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) {
countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties);
Expand All @@ -260,14 +258,14 @@ public static void main(String[] args) {
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ CONFIG_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
cnMapProperties);
propertiesFileUpdate(
targetDirString.toString()
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ DATA_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
dnMapProperties);
}

Expand All @@ -293,7 +291,7 @@ public static void main(String[] args) {
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ CONFIG_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
cnMapProperties);
} else if (nodeTypeParam.equalsIgnoreCase("datanode")) {
countNodeBack(targetDirString.toString(), copyMap);
Expand All @@ -313,7 +311,7 @@ public static void main(String[] args) {
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ DATA_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
dnMapProperties);
} else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) {
countNodeBack(targetDirString.toString(), copyMap);
Expand All @@ -335,14 +333,14 @@ public static void main(String[] args) {
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ CONFIG_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
cnMapProperties);
propertiesFileUpdate(
targetDirString.toString()
+ File.separatorChar
+ "conf"
+ File.separatorChar
+ DATA_NODE_CONF_NAME,
+ CommonConfig.SYSTEM_CONFIG_NAME,
dnMapProperties);
}
}
Expand Down Expand Up @@ -444,10 +442,9 @@ private static void ioTDBDataBack(

private static void countNodeBack(String targetDirString, Map<String, String> copyMap) {
File sourceDir = new File(sourcePath);
Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME);
initDataNodeProperties(dataProperties);
Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME);
initConfigNodeProperties(configProperties);
Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
initDataNodeProperties(properties);
initConfigNodeProperties(properties);

copyMap.put(
sourceDir.getAbsolutePath() + File.separatorChar + ".env",
Expand Down Expand Up @@ -495,7 +492,7 @@ private static void countDataNodeFile(
Map<String, String> copyMap,
Map<String, String> dnDataDirsMap,
Map<String, String> dnMapProperties) {
Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME);
Properties dataProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
initDataNodeProperties(dataProperties);

String dnSystemDir = dataProperties.getProperty("dn_system_dir");
Expand Down Expand Up @@ -593,7 +590,7 @@ private static String isObjectStorage(String dnDataDirs) {

private static void countConfigNodeFile(
String targetDirString, Map<String, String> copyMap, Map<String, String> cnMapProperties) {
Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME);
Properties configProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME);
initConfigNodeProperties(configProperties);

String bakCnSystemDir = targetDirString + File.separatorChar + DEFAULT_CN_SYSTEM_DIR;
Expand Down
4 changes: 2 additions & 2 deletions iotdb-core/confignode/src/assembly/confignode.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@
</fileSets>
<files>
<file>
<source>${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties</source>
<destName>conf/iotdb-common.properties</destName>
<source>${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties</source>
<destName>conf/iotdb-system.properties</destName>
</file>
<file>
<source>${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh</source>
Expand Down
Loading

0 comments on commit d921bdb

Please sign in to comment.