add property for tmp directory

jetoile · Oct 17, 2019 · 55e7b9b · 55e7b9b
1 parent 553edfd
commit 55e7b9b
Show file tree

Hide file tree

Showing 58 changed files with 648 additions and 547 deletions.
diff --git a/hadoop-unit-alluxio/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-alluxio/src/test/resources/hadoop-unit-default.properties
@@ -1,18 +1,20 @@
 #HADOOP_HOME=/opt/hadoop
 
+tmp.dir.path=/tmp
+
 # Zookeeper
-zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.temp.dir=/embedded_zk
 zookeeper.host=127.0.0.1
 zookeeper.port=22010
 
 # Hive
-hive.scratch.dir=/tmp/hive_scratch_dir
+hive.scratch.dir=/hive_scratch_dir
 hive.warehouse.dir=/tmp/warehouse_dir
 
 # Hive Metastore
 hive.metastore.hostname=localhost
 hive.metastore.port=20102
-hive.metastore.derby.db.dir=metastore_db
+hive.metastore.derby.db.dir=/metastore_db
 
 # Hive Server2
 hive.server2.hostname=localhost
@@ -27,7 +29,7 @@ hive.test.table.name=test_table
 hdfs.namenode.host=localhost
 hdfs.namenode.port=20112
 hdfs.namenode.http.port=50070
-hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.temp.dir=/embedded_hdfs
 hdfs.num.datanodes=1
 hdfs.enable.permissions=false
 hdfs.format=true
@@ -42,7 +44,7 @@ hdfs.test.string=TESTING
 hbase.master.port=25111
 hbase.master.info.port=-1
 hbase.num.region.servers=1
-hbase.root.dir=/tmp/embedded_hbase
+hbase.root.dir=/embedded_hbase
 hbase.znode.parent=/hbase-unsecure
 hbase.wal.replication.enabled=false
 
@@ -60,7 +62,7 @@ kafka.port=20111
 kafka.test.topic=testtopic
 kafka.test.message.count=10
 kafka.test.broker.id=1
-kafka.test.temp.dir=embedded_kafka
+kafka.test.temp.dir=/embedded_kafka
 
 #SolR + SolRCloud
 solr.dir=solr
@@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false
 mr.job.history.address=localhost:37005
 
 # Oozie
-oozie.test.dir=/tmp/embedded_oozie
-oozie.home.dir=/tmp/oozie_home
+oozie.test.dir=/embedded_oozie
+oozie.home.dir=/oozie_home
 oozie.username=blah
 oozie.groupname=testgroup
 oozie.hdfs.share.lib.dir=/tmp/share_lib
@@ -104,7 +106,7 @@ oozie.host=localhost
 # Neo4j
 neo4j.ip=127.0.0.1
 neo4j.port=13533
-neo4j.temp.dir=/tmp/embedded_neo4j
+neo4j.temp.dir=/embedded_neo4j
 
 # Alluxio
 #alluxio.work.dir=/tmp/alluxio

diff --git a/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java b/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java
@@ -40,6 +40,7 @@ public class CassandraBootstrap implements Bootstrap {
     private CassandraShutDownHook shutdownHook;
     private int port;
     private String ip;
+    private String tmpDirPath;
 
     public CassandraBootstrap() {
         try {
@@ -73,6 +74,7 @@ public String getProperties() {
     private void loadConfig() {
         port = configuration.getInt(CassandraConfig.CASSANDRA_PORT_KEY);
         ip = configuration.getString(CassandraConfig.CASSANDRA_IP_KEY);
+        tmpDirPath = getTmpDirPath(configuration, CassandraConfig.CASSANDRA_TEMP_DIR_KEY);
     }
 
     @Override
@@ -83,14 +85,17 @@ public void loadConfig(Map<String, String> configs) {
         if (StringUtils.isNotEmpty(configs.get(CassandraConfig.CASSANDRA_IP_KEY))) {
             ip = configs.get(CassandraConfig.CASSANDRA_IP_KEY);
         }
+        if (StringUtils.isNotEmpty(configs.get(CassandraConfig.CASSANDRA_TEMP_DIR_KEY))) {
+            tmpDirPath = getTmpDirPath(configs, CassandraConfig.CASSANDRA_TEMP_DIR_KEY);
+        }
     }
 
     private void build() throws IOException {
-        Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY)));
-        Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/data"));
-        Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog"));
-        Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches"));
-        Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints"));
+        Files.createDirectory(Paths.get(tmpDirPath));
+        Files.createDirectory(Paths.get(tmpDirPath + "/data"));
+        Files.createDirectory(Paths.get(tmpDirPath + "/commitlog"));
+        Files.createDirectory(Paths.get(tmpDirPath + "/saved_caches"));
+        Files.createDirectory(Paths.get(tmpDirPath + "/hints"));
 
         shutdownHook = new CassandraShutDownHook();
 
@@ -100,10 +105,10 @@ private void build() throws IOException {
                 .withBroadcastAddress(ip)
                 .withBroadcastRpcAddress(ip)
                 .withCQLPort(port)
-                .withDataFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/data")
-                .withCommitLogFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog")
-                .withSavedCachesFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches")
-                .withHintsFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints")
+                .withDataFolder(tmpDirPath + "/data")
+                .withCommitLogFolder(tmpDirPath + "/commitlog")
+                .withSavedCachesFolder(tmpDirPath + "/saved_caches")
+                .withHintsFolder(tmpDirPath + "/hints")
                 .cleanDataFilesAtStartup(true)
                 .withShutdownHook(shutdownHook)
                 .cleanDataFilesAtStartup(true)
@@ -147,9 +152,9 @@ public Bootstrap stop() {
 
     private void cleanup() {
         try {
-            FileUtils.deleteDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY)).toFile());
+            FileUtils.deleteDirectory(Paths.get(tmpDirPath).toFile());
         } catch (IOException e) {
-            LOGGER.error("unable to delete {}", configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY), e);
+            LOGGER.error("unable to delete {}", tmpDirPath, e);
         }
     }
 

diff --git a/hadoop-unit-cassandra/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-cassandra/src/main/resources/hadoop-unit-default.properties
@@ -1,18 +1,20 @@
 #HADOOP_HOME=/opt/hadoop
 
+tmp.dir.path=/tmp
+
 # Zookeeper
-zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.temp.dir=/embedded_zk
 zookeeper.host=127.0.0.1
 zookeeper.port=22010
 
 # Hive
-hive.scratch.dir=/tmp/hive_scratch_dir
+hive.scratch.dir=/hive_scratch_dir
 hive.warehouse.dir=/tmp/warehouse_dir
 
 # Hive Metastore
 hive.metastore.hostname=localhost
 hive.metastore.port=20102
-hive.metastore.derby.db.dir=metastore_db
+hive.metastore.derby.db.dir=/metastore_db
 
 # Hive Server2
 hive.server2.hostname=localhost
@@ -27,7 +29,7 @@ hive.test.table.name=test_table
 hdfs.namenode.host=localhost
 hdfs.namenode.port=20112
 hdfs.namenode.http.port=50070
-hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.temp.dir=/embedded_hdfs
 hdfs.num.datanodes=1
 hdfs.enable.permissions=false
 hdfs.format=true
@@ -42,7 +44,7 @@ hdfs.test.string=TESTING
 hbase.master.port=25111
 hbase.master.info.port=-1
 hbase.num.region.servers=1
-hbase.root.dir=/tmp/embedded_hbase
+hbase.root.dir=/embedded_hbase
 hbase.znode.parent=/hbase-unsecure
 hbase.wal.replication.enabled=false
 
@@ -60,7 +62,7 @@ kafka.port=20111
 kafka.test.topic=testtopic
 kafka.test.message.count=10
 kafka.test.broker.id=1
-kafka.test.temp.dir=embedded_kafka
+kafka.test.temp.dir=/embedded_kafka
 
 #SolR + SolRCloud
 solr.dir=solr
@@ -91,9 +93,9 @@ yarn.use.in.jvm.container.executor=false
 mr.job.history.address=localhost:37005
 
 # Oozie
-oozie.tmp.dir=/tmp/oozie_tmp
-oozie.test.dir=/tmp/embedded_oozie
-oozie.home.dir=/tmp/oozie_home
+oozie.tmp.dir=/oozie_tmp
+oozie.test.dir=/embedded_oozie
+oozie.home.dir=/oozie_home
 oozie.username=blah
 oozie.groupname=testgroup
 oozie.hdfs.share.lib.dir=/tmp/share_lib
@@ -114,7 +116,7 @@ mongo.collection.name=test_collection
 # Cassandra
 cassandra.ip=127.0.0.1
 cassandra.port=13433
-cassandra.temp.dir=/tmp/embedded_cassandra
+cassandra.temp.dir=/embedded_cassandra
 
 # ElasticSearch
 elasticsearch.version=5.4.3
@@ -127,4 +129,4 @@ elasticsearch.cluster.name=elasticsearch
 # Neo4j
 neo4j.ip=127.0.0.1
 neo4j.port=13533
-neo4j.temp.dir=/tmp/embedded_neo4j
+neo4j.temp.dir=/embedded_neo4j
diff --git a/hadoop-unit-cassandra/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-cassandra/src/test/resources/hadoop-unit-default.properties
@@ -1,32 +1,20 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 #HADOOP_HOME=/opt/hadoop
 
+tmp.dir.path=/tmp
+
 # Zookeeper
-zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.temp.dir=/embedded_zk
 zookeeper.host=127.0.0.1
 zookeeper.port=22010
 
 # Hive
-hive.scratch.dir=/tmp/hive_scratch_dir
+hive.scratch.dir=/hive_scratch_dir
 hive.warehouse.dir=/tmp/warehouse_dir
 
 # Hive Metastore
 hive.metastore.hostname=localhost
 hive.metastore.port=20102
-hive.metastore.derby.db.dir=metastore_db
+hive.metastore.derby.db.dir=/metastore_db
 
 # Hive Server2
 hive.server2.hostname=localhost
@@ -41,7 +29,7 @@ hive.test.table.name=test_table
 hdfs.namenode.host=localhost
 hdfs.namenode.port=20112
 hdfs.namenode.http.port=50070
-hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.temp.dir=/embedded_hdfs
 hdfs.num.datanodes=1
 hdfs.enable.permissions=false
 hdfs.format=true
@@ -56,7 +44,7 @@ hdfs.test.string=TESTING
 hbase.master.port=25111
 hbase.master.info.port=-1
 hbase.num.region.servers=1
-hbase.root.dir=/tmp/embedded_hbase
+hbase.root.dir=/embedded_hbase
 hbase.znode.parent=/hbase-unsecure
 hbase.wal.replication.enabled=false
 
@@ -74,7 +62,7 @@ kafka.port=20111
 kafka.test.topic=testtopic
 kafka.test.message.count=10
 kafka.test.broker.id=1
-kafka.test.temp.dir=embedded_kafka
+kafka.test.temp.dir=/embedded_kafka
 
 #SolR + SolRCloud
 solr.dir=solr
@@ -87,6 +75,9 @@ solr.collection.name=collection1
 solr.cloud.port=8983
 
 
+
+
+
 # YARN
 yarn.num.node.managers=1
 yarn.num.local.dirs=1
@@ -102,8 +93,9 @@ yarn.use.in.jvm.container.executor=false
 mr.job.history.address=localhost:37005
 
 # Oozie
-oozie.test.dir=/tmp/embedded_oozie
-oozie.home.dir=/tmp/oozie_home
+oozie.tmp.dir=/oozie_tmp
+oozie.test.dir=/embedded_oozie
+oozie.home.dir=/oozie_home
 oozie.username=blah
 oozie.groupname=testgroup
 oozie.hdfs.share.lib.dir=/tmp/share_lib
@@ -124,9 +116,17 @@ mongo.collection.name=test_collection
 # Cassandra
 cassandra.ip=127.0.0.1
 cassandra.port=13433
-cassandra.temp.dir=/tmp/embedded_cassandra
+cassandra.temp.dir=/embedded_cassandra
+
+# ElasticSearch
+elasticsearch.version=5.4.3
+elasticsearch.ip=127.0.0.1
+elasticsearch.http.port=14433
+elasticsearch.tcp.port=14533
+elasticsearch.index.name=test_index
+elasticsearch.cluster.name=elasticsearch
 
 # Neo4j
 neo4j.ip=127.0.0.1
 neo4j.port=13533
-neo4j.temp.dir=/tmp/embedded_neo4j
+neo4j.temp.dir=/embedded_neo4j
diff --git a/...-unit-client/hadoop-unit-client-alluxio/src/test/resources/hadoop-unit-default.properties b/...-unit-client/hadoop-unit-client-alluxio/src/test/resources/hadoop-unit-default.properties
@@ -1,18 +1,18 @@
 #HADOOP_HOME=/opt/hadoop
 
 # Zookeeper
-zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.temp.dir=/embedded_zk
 zookeeper.host=127.0.0.1
 zookeeper.port=22010
 
 # Hive
-hive.scratch.dir=/tmp/hive_scratch_dir
+hive.scratch.dir=/hive_scratch_dir
 hive.warehouse.dir=/tmp/warehouse_dir
 
 # Hive Metastore
 hive.metastore.hostname=localhost
 hive.metastore.port=20102
-hive.metastore.derby.db.dir=metastore_db
+hive.metastore.derby.db.dir=/metastore_db
 
 # Hive Server2
 hive.server2.hostname=localhost
@@ -27,7 +27,7 @@ hive.test.table.name=test_table
 hdfs.namenode.host=localhost
 hdfs.namenode.port=20112
 hdfs.namenode.http.port=50070
-hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.temp.dir=/embedded_hdfs
 hdfs.num.datanodes=1
 hdfs.enable.permissions=false
 hdfs.format=true
@@ -42,7 +42,7 @@ hdfs.test.string=TESTING
 hbase.master.port=25111
 hbase.master.info.port=-1
 hbase.num.region.servers=1
-hbase.root.dir=/tmp/embedded_hbase
+hbase.root.dir=/embedded_hbase
 hbase.znode.parent=/hbase-unsecure
 hbase.wal.replication.enabled=false
 
@@ -60,7 +60,7 @@ kafka.port=20111
 kafka.test.topic=testtopic
 kafka.test.message.count=10
 kafka.test.broker.id=1
-kafka.test.temp.dir=embedded_kafka
+kafka.test.temp.dir=/embedded_kafka
 
 #SolR + SolRCloud
 solr.dir=solr
@@ -88,8 +88,8 @@ yarn.use.in.jvm.container.executor=false
 mr.job.history.address=localhost:37005
 
 # Oozie
-oozie.test.dir=/tmp/embedded_oozie
-oozie.home.dir=/tmp/oozie_home
+oozie.test.dir=/embedded_oozie
+oozie.home.dir=/oozie_home
 oozie.username=blah
 oozie.groupname=testgroup
 oozie.hdfs.share.lib.dir=/tmp/share_lib
@@ -104,7 +104,7 @@ oozie.host=localhost
 # Neo4j
 neo4j.ip=127.0.0.1
 neo4j.port=13533
-neo4j.temp.dir=/tmp/embedded_neo4j
+neo4j.temp.dir=/embedded_neo4j
 
 # Alluxio
 alluxio.work.dir=/tmp/alluxio