[Improve][system config] improve config file

config/hazelcast-client.yaml

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast-client:
+  cluster-name: seatunnel
+  properties:
+    hazelcast.logging.type: log4j2
+  network:
+    cluster-members:
+      - localhost:5801

config/hazelcast.yaml

@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast:
+  cluster-name: seatunnel
+  network:
+    rest-api:
+      enabled: true
+      endpoint-groups:
+        CLUSTER_WRITE:
+          enabled: true
+    join:
+      tcp-ip:
+        enabled: true
+        member-list:
+          - localhost
+    port:
+      auto-increment: false
+      port: 5801
+  properties:
+    hazelcast.invocation.max.retry.count: 20
+    hazelcast.tcp.join.port.try.count: 30
+    hazelcast.logging.type: log4j2
+

config/log4j2.properties

@@ -25,11 +25,11 @@ property.file_ttl = 7d
 rootLogger.level = INFO
 
 ############################ log output to console #############################
-rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
-rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
+#rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
+#rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
 ############################ log output to console #############################
 ############################ log output to file    #############################
-#rootLogger.appenderRef.file.ref = fileAppender
+rootLogger.appenderRef.file.ref = fileAppender
 ############################ log output to file    #############################
 
 appender.consoleStdout.name = consoleStdoutAppender
@@ -75,4 +75,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny
 appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
 appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
 appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

config/log4j2_client.properties

@@ -0,0 +1,78 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+property.file_path = ${sys:seatunnel.logs.path:-/tmp/seatunnel/logs}
+property.file_name = ${sys:seatunnel.logs.file_name:-seatunnel}
+property.file_split_size = 100MB
+property.file_count = 100
+property.file_ttl = 7d
+
+rootLogger.level = INFO
+
+############################ log output to console #############################
+rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
+rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
+############################ log output to console #############################
+############################ log output to file    #############################
+#rootLogger.appenderRef.file.ref = fileAppender
+############################ log output to file    #############################
+
+appender.consoleStdout.name = consoleStdoutAppender
+appender.consoleStdout.type = CONSOLE
+appender.consoleStdout.target = SYSTEM_OUT
+appender.consoleStdout.layout.type = PatternLayout
+appender.consoleStdout.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
+appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter
+appender.consoleStdout.filter.acceptLtWarn.level = WARN
+appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY
+appender.consoleStdout.filter.acceptLtWarn.onMismatch = ACCEPT
+
+appender.consoleStderr.name = consoleStderrAppender
+appender.consoleStderr.type = CONSOLE
+appender.consoleStderr.target = SYSTEM_ERR
+appender.consoleStderr.layout.type = PatternLayout
+appender.consoleStderr.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
+appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter
+appender.consoleStderr.filter.acceptGteWarn.level = WARN
+appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT
+appender.consoleStderr.filter.acceptGteWarn.onMismatch = DENY
+
+appender.file.name = fileAppender
+appender.file.type = RollingFile
+appender.file.fileName = ${file_path}/${file_name}.log
+appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i
+appender.file.append = true
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
+appender.file.policies.type = Policies
+appender.file.policies.time.type = TimeBasedTriggeringPolicy
+appender.file.policies.time.modulate = true
+appender.file.policies.size.type = SizeBasedTriggeringPolicy
+appender.file.policies.size.size = ${file_split_size}
+appender.file.strategy.type = DefaultRolloverStrategy
+appender.file.strategy.fileIndex = nomax
+appender.file.strategy.action.type = Delete
+appender.file.strategy.action.basepath = ${file_path}
+appender.file.strategy.action.maxDepth = 1
+appender.file.strategy.action.condition.type = IfFileName
+appender.file.strategy.action.condition.glob = ${file_name}.log*
+appender.file.strategy.action.condition.nested_condition.type = IfAny
+appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
+appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
+appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

config/seatunnel.yaml

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+seatunnel:
+  engine:
+    backup-count: 1
+    print-execution-info-interval: 10
+    slot-service:
+      dynamic-slot: true
+    checkpoint:
+      interval: 300000
+      timeout: 10000
+      max-concurrent: 5
+      tolerable-failure: 2
+      storage:
+        type: hdfs
+        max-retained: 3
+        namespace: /tmp/seatunnel/checkpoint_snapshot
+        plugin-config:
+          storage-type: hdfs
+          fs.defaultFS: /tmp/ # Ensure that the directory has written permission

config/v2.batch.config.template

@@ -22,6 +22,7 @@ env {
   # You can set SeaTunnel environment configuration here
   execution.parallelism = 2
   job.mode = "BATCH"
+  checkpoint.interval = 10000
   #execution.checkpoint.interval = 10000
   #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
 }
@@ -44,20 +45,10 @@ source {
   # please go to https://seatunnel.apache.org/docs/category/source-v2
 }
 
-transform {
-  sql {
-    sql = "select name,age from fake"
-  }
-
-  # If you would like to get more information about how to configure Seatunnel and see full list of transform plugins,
-  # please go to https://seatunnel.apache.org/docs/category/transform
-}
-
 sink {
   Console {
-    parallelism = 3
   }
 
   # If you would like to get more information about how to configure Seatunnel and see full list of sink plugins,
   # please go to https://seatunnel.apache.org/docs/category/sink-v2
-}
+}

config/v2.streaming.conf.template

@@ -22,6 +22,7 @@ env {
   # You can set flink configuration here
   execution.parallelism = 2
   job.mode = "STREAMING"
+  checkpoint.interval = 2000
   #execution.checkpoint.interval = 10000
   #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
 }
@@ -44,20 +45,10 @@ source {
   # please go to https://seatunnel.apache.org/docs/category/source-v2
 }
 
-transform {
-  sql {
-    sql = "select name,age from fake"
-  }
-
-  # If you would like to get more information about how to configure Seatunnel and see full list of transform plugins,
-  # please go to https://seatunnel.apache.org/docs/category/transform
-}
-
 sink {
   Console {
-    parallelism = 3
   }
 
   # If you would like to get more information about how to configure Seatunnel and see full list of sink plugins,
   # please go to https://seatunnel.apache.org/docs/category/sink-v2
-}
+}

docs/en/seatunnel-engine/checkpoint-storage.md

@@ -116,6 +116,6 @@ seatunnel:
         max-retained: 3
         plugin-config:
           storage-type: hdfs
-            fs.defaultFS: /tmp/ # Ensure that the directory has written permission 
+          fs.defaultFS: /tmp/ # Ensure that the directory has written permission 
 
 ```

seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh

@@ -81,7 +81,7 @@ JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.a
 if [ -e "${CONF_DIR}/log4j2.properties" ]; then
   JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.configurationFile=${CONF_DIR}/log4j2.properties"
   JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${APP_DIR}/logs"
-  JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.file_name=seatunnel-starter-server"
+  JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.file_name=seatunnel-engine-server"
 fi
 
 echo "JAVA_OPTS: ${JAVA_OPTS}"

seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh

@@ -82,8 +82,8 @@ JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.config=${SEATUNNEL_CONFIG}"
 JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.config=${HAZELCAST_CONFIG}"
 
 # Log4j2 Config
-if [ -e "${CONF_DIR}/log4j2.properties" ]; then
-  JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.configurationFile=${CONF_DIR}/log4j2.properties"
+if [ -e "${CONF_DIR}/log4j2_client.properties" ]; then
+  JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.configurationFile=${CONF_DIR}/log4j2_client.properties"
   JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${APP_DIR}/logs"
   if [[ $args == *" -e local"* || $args == *" --deploy-mode local"* ]]; then
     JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.file_name=seatunnel-starter-client-$((`date '+%s'`*1000+`date '+%N'`/1000000))"

seatunnel-dist/src/main/assembly/assembly-bin-ci.xml

@@ -69,15 +69,6 @@
             </includes>
             <fileMode>0755</fileMode>
         </fileSet>
-        <!--Engine Config File-->
-        <fileSet>
-            <directory>../seatunnel-engine/seatunnel-engine-common/src/main/resources</directory>
-            <includes>
-                <include>*.yaml</include>
-            </includes>
-            <outputDirectory>/config</outputDirectory>
-            <fileMode>0644</fileMode>
-        </fileSet>
         <!--Licenses And NOTICE-->
         <fileSet>
             <directory>release-docs</directory>

seatunnel-dist/src/main/assembly/assembly-bin.xml

@@ -70,15 +70,6 @@
             </includes>
             <fileMode>0755</fileMode>
         </fileSet>
-        <!--Engine Config File-->
-        <fileSet>
-            <directory>../seatunnel-engine/seatunnel-engine-common/src/main/resources</directory>
-            <includes>
-                <include>*.yaml</include>
-            </includes>
-            <outputDirectory>/config</outputDirectory>
-            <fileMode>0644</fileMode>
-        </fileSet>
         <!--Licenses And NOTICE-->
         <fileSet>
             <directory>release-docs</directory>

seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/java/org/apache/seatunnel/e2e/connector/neo4j/Neo4jIT.java

@@ -64,7 +64,7 @@ public class Neo4jIT extends TestSuiteBase implements TestResource {
     private static final int HTTP_PORT = 7474;
     private static final int BOLT_PORT = 7687;
     private static final String CONTAINER_NEO4J_USERNAME = "neo4j";
-    private static final String CONTAINER_NEO4J_PASSWORD = "1234";
+    private static final String CONTAINER_NEO4J_PASSWORD = "Test@12343";
     private static final URI CONTAINER_URI = URI.create("neo4j://localhost:" + BOLT_PORT);
 
     private GenericContainer<?> container;

seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf

@@ -33,7 +33,7 @@ source {
     Neo4j {
         uri = "neo4j://neo4j-host:7687"
         username = "neo4j"
-        password = "1234"
+        password = "Test@12343"
         database = "neo4j"
 
         max_transaction_retry_time = 1
@@ -67,7 +67,7 @@ sink {
   Neo4j {
       uri = "neo4j://neo4j-host:7687"
       username = "neo4j"
-      password = "1234"
+      password = "Test@12343"
       database = "neo4j"
 
       max_transaction_retry_time = 1

seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast-client.yaml

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast-client:
+  cluster-name: seatunnel
+
+  network:
+    cluster-members:
+      - localhost:5801
+      - localhost:5802
+      - localhost:5803
+      - localhost:5804
+      - localhost:5805
+      - localhost:5806
+      - localhost:5807
+      - localhost:5808
+      - localhost:5809
+      - localhost:5810
+      - localhost:5811
+      - localhost:5812
+      - localhost:5813
+      - localhost:5814
+      - localhost:5815