From 55e7b9b06c26578348d45d92ae044d5a1ce2dd0f Mon Sep 17 00:00:00 2001 From: jetoile Date: Thu, 17 Oct 2019 17:33:19 +0200 Subject: [PATCH] add property for tmp directory --- .../resources/hadoop-unit-default.properties | 20 ++++---- .../component/CassandraBootstrap.java | 27 ++++++----- .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 48 +++++++++---------- .../resources/hadoop-unit-default.properties | 18 +++---- .../jetoile/hadoopunit/HadoopUnitConfig.java | 1 + .../hadoopunit/component/Bootstrap.java | 10 ++++ .../resources/hadoop-unit-default.properties | 18 +++---- .../resources/hadoop-unit-default.properties | 30 ++++++------ .../component/ConfluentKafkaBootstrap.java | 6 +-- .../resources/hadoop-unit-default.properties | 30 ++++++------ .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 40 ++++++---------- .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 43 +++++++++++++---- .../hadoopunit/component/HdfsBootstrap.java | 4 +- .../resources/hadoop-unit-default.properties | 20 ++++---- .../resources/hadoop-unit-default.properties | 20 ++++---- .../component/HiveMetastoreBootstrap.java | 8 ++-- .../component/HiveServer2Bootstrap.java | 8 ++-- .../resources/hadoop-unit-default.properties | 20 ++++---- .../resources/hadoop-unit-default.properties | 22 +++++---- .../hadoopunit/component/KafkaBootstrap.java | 4 +- .../resources/hadoop-unit-default.properties | 20 ++++---- .../resources/hadoop-unit-default.properties | 20 ++++---- .../hadoopunit/component/KnoxBootstrap.java | 4 +- .../resources/hadoop-unit-default.properties | 20 ++++---- .../resources/hadoop-unit-default.properties | 28 ++++++----- .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 20 ++++---- .../hadoopunit/component/Neo4jBootstrap.java | 8 ++-- .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 21 ++++---- .../hadoopunit/component/OozieBootstrap.java | 8 ++-- .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 21 ++++---- .../hadoopunit/component/RedisBootstrap.java | 4 +- .../resources/hadoop-unit-default.properties | 26 +++++----- .../resources/hadoop-unit-default.properties | 25 +++++----- .../resources/hadoop-unit-default.properties | 23 ++++----- .../resources/hadoop-unit-default.properties | 19 ++++---- .../resources/hadoop-unit-default.properties | 19 ++++---- .../resources/hadoop-unit-default.properties | 19 ++++---- .../resources/hadoop-unit-default.properties | 30 ++++++------ .../resources/hadoop-unit-default.properties | 28 ++++++----- .../resources/hadoop-unit-default.properties | 28 ++++++----- .../component/ZookeeperBootstrap.java | 4 +- .../resources/hadoop-unit-default.properties | 20 ++++---- .../resources/hadoop-unit-default.properties | 20 ++++---- .../resources/hadoop-unit-default.properties | 24 +++++----- .../resources/hadoop-unit-default.properties | 30 ++++++------ .../resources/hadoop-unit-default.properties | 22 +++++---- .../resources/hadoop-unit-default.properties | 22 +++++---- .../resources/hadoop-unit-default.properties | 26 +++++----- .../resources/hadoop-unit-default.properties | 22 +++++---- .../resources/hadoop-unit-default.properties | 28 ++++++----- .../resources/hadoop-unit-default.properties | 22 +++++---- .../resources/hadoop-unit-default.properties | 23 ++++----- 58 files changed, 648 insertions(+), 547 deletions(-) diff --git a/hadoop-unit-alluxio/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-alluxio/src/test/resources/hadoop-unit-default.properties index 20482db7..e12dd621 100644 --- a/hadoop-unit-alluxio/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-alluxio/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,7 +106,7 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # Alluxio #alluxio.work.dir=/tmp/alluxio diff --git a/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java b/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java index 2fc4453b..a6597152 100644 --- a/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java +++ b/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java @@ -40,6 +40,7 @@ public class CassandraBootstrap implements Bootstrap { private CassandraShutDownHook shutdownHook; private int port; private String ip; + private String tmpDirPath; public CassandraBootstrap() { try { @@ -73,6 +74,7 @@ public String getProperties() { private void loadConfig() { port = configuration.getInt(CassandraConfig.CASSANDRA_PORT_KEY); ip = configuration.getString(CassandraConfig.CASSANDRA_IP_KEY); + tmpDirPath = getTmpDirPath(configuration, CassandraConfig.CASSANDRA_TEMP_DIR_KEY); } @Override @@ -83,14 +85,17 @@ public void loadConfig(Map configs) { if (StringUtils.isNotEmpty(configs.get(CassandraConfig.CASSANDRA_IP_KEY))) { ip = configs.get(CassandraConfig.CASSANDRA_IP_KEY); } + if (StringUtils.isNotEmpty(configs.get(CassandraConfig.CASSANDRA_TEMP_DIR_KEY))) { + tmpDirPath = getTmpDirPath(configs, CassandraConfig.CASSANDRA_TEMP_DIR_KEY); + } } private void build() throws IOException { - Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY))); - Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/data")); - Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog")); - Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches")); - Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints")); + Files.createDirectory(Paths.get(tmpDirPath)); + Files.createDirectory(Paths.get(tmpDirPath + "/data")); + Files.createDirectory(Paths.get(tmpDirPath + "/commitlog")); + Files.createDirectory(Paths.get(tmpDirPath + "/saved_caches")); + Files.createDirectory(Paths.get(tmpDirPath + "/hints")); shutdownHook = new CassandraShutDownHook(); @@ -100,10 +105,10 @@ private void build() throws IOException { .withBroadcastAddress(ip) .withBroadcastRpcAddress(ip) .withCQLPort(port) - .withDataFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/data") - .withCommitLogFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog") - .withSavedCachesFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches") - .withHintsFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints") + .withDataFolder(tmpDirPath + "/data") + .withCommitLogFolder(tmpDirPath + "/commitlog") + .withSavedCachesFolder(tmpDirPath + "/saved_caches") + .withHintsFolder(tmpDirPath + "/hints") .cleanDataFilesAtStartup(true) .withShutdownHook(shutdownHook) .cleanDataFilesAtStartup(true) @@ -147,9 +152,9 @@ public Bootstrap stop() { private void cleanup() { try { - FileUtils.deleteDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY)).toFile()); + FileUtils.deleteDirectory(Paths.get(tmpDirPath).toFile()); } catch (IOException e) { - LOGGER.error("unable to delete {}", configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY), e); + LOGGER.error("unable to delete {}", tmpDirPath, e); } } diff --git a/hadoop-unit-cassandra/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-cassandra/src/main/resources/hadoop-unit-default.properties index 2391aa5a..42fcd366 100644 --- a/hadoop-unit-cassandra/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-cassandra/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,9 +93,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -114,7 +116,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -127,4 +129,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-cassandra/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-cassandra/src/test/resources/hadoop-unit-default.properties index fc0ba0d1..42fcd366 100644 --- a/hadoop-unit-cassandra/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-cassandra/src/test/resources/hadoop-unit-default.properties @@ -1,32 +1,20 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -41,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -56,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -74,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -87,6 +75,9 @@ solr.collection.name=collection1 solr.cloud.port=8983 + + + # YARN yarn.num.node.managers=1 yarn.num.local.dirs=1 @@ -102,8 +93,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -124,9 +116,17 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra + +# ElasticSearch +elasticsearch.version=5.4.3 +elasticsearch.ip=127.0.0.1 +elasticsearch.http.port=14433 +elasticsearch.tcp.port=14533 +elasticsearch.index.name=test_index +elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-client/hadoop-unit-client-alluxio/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-client/hadoop-unit-client-alluxio/src/test/resources/hadoop-unit-default.properties index 763041af..b16b9f6e 100644 --- a/hadoop-unit-client/hadoop-unit-client-alluxio/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-client/hadoop-unit-client-alluxio/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,18 @@ #HADOOP_HOME=/opt/hadoop # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +27,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +42,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +60,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +88,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,7 +104,7 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # Alluxio alluxio.work.dir=/tmp/alluxio diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUnitConfig.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUnitConfig.java index f106109f..e5b9f260 100644 --- a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUnitConfig.java +++ b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUnitConfig.java @@ -17,4 +17,5 @@ public class HadoopUnitConfig { // Props file public static final String DEFAULT_PROPS_FILE = "hadoop-unit-default.properties"; + public static final String TMP_DIR_PATH_KEY = "tmp.dir.path"; } diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/component/Bootstrap.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/component/Bootstrap.java index ece7432f..22326483 100644 --- a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/component/Bootstrap.java +++ b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/component/Bootstrap.java @@ -16,6 +16,8 @@ import fr.jetoile.hadoopunit.ComponentMetadata; +import fr.jetoile.hadoopunit.HadoopUnitConfig; +import org.apache.commons.configuration.Configuration; import java.util.Map; @@ -26,6 +28,14 @@ public interface Bootstrap { String getProperties(); + default String getTmpDirPath(Configuration configuration, String componentTmpPathKey) { + return configuration.getString(HadoopUnitConfig.TMP_DIR_PATH_KEY, "/tmp") + configuration.getString(componentTmpPathKey); + } + + default String getTmpDirPath(Map configs, String componentTmpPathKey) { + return configs.getOrDefault(HadoopUnitConfig.TMP_DIR_PATH_KEY, "/tmp") + configs.get(componentTmpPathKey); + } + /** * Allow to override configurations when using maven plugin * diff --git a/hadoop-unit-commons/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-commons/src/test/resources/hadoop-unit-default.properties index 219d8ff7..5f8c9ecf 100644 --- a/hadoop-unit-commons/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-commons/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,18 @@ #HADOOP_HOME=/opt/hadoop # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +27,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +42,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +60,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +88,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +104,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-confluent-rest/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-confluent-rest/src/main/resources/hadoop-unit-default.properties index 736fb25b..b55a6dfd 100644 --- a/hadoop-unit-confluent-rest/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-confluent-rest/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -99,9 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -133,19 +135,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase #knox.service=namenode,webhdfs,webhbase,oozie @@ -167,7 +169,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE @@ -181,7 +183,7 @@ confluent.schemaregistry.host=127.0.0.1 confluent.schemaregistry.kafkastore.topic=_schema confluent.schemaregistry.debug=false -confluent.kafka.log.dirs=/tmp/kafka-logs +confluent.kafka.log.dirs=/kafka-logs confluent.kafka.broker.id=0 confluent.kafka.port=22222 confluent.kafka.host=127.0.0.1 diff --git a/hadoop-unit-confluent/src/main/java/fr/jetoile/hadoopunit/component/ConfluentKafkaBootstrap.java b/hadoop-unit-confluent/src/main/java/fr/jetoile/hadoopunit/component/ConfluentKafkaBootstrap.java index 7a72e364..ba64d024 100644 --- a/hadoop-unit-confluent/src/main/java/fr/jetoile/hadoopunit/component/ConfluentKafkaBootstrap.java +++ b/hadoop-unit-confluent/src/main/java/fr/jetoile/hadoopunit/component/ConfluentKafkaBootstrap.java @@ -79,7 +79,7 @@ public String getProperties() { public void loadConfig() { kafkaConfig.put("zookeeper.connect", configuration.getString(ZookeeperConfig.ZOOKEEPER_HOST_KEY) + ":" + configuration.getString(ZookeeperConfig.ZOOKEEPER_PORT_KEY)); - kafkaConfig.put("log.dirs", configuration.getString(ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY)); + kafkaConfig.put("log.dirs", getTmpDirPath(configuration, ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY)); kafkaConfig.put("broker.id", configuration.getString(ConfluentConfig.CONFLUENT_KAFKA_BROKER_ID_KEY)); // kafkaConfig.put("advertised.listeners", "PLAINTEXT://localhost:22222"); kafkaConfig.put("advertised.host.name", configuration.getString(ConfluentConfig.CONFLUENT_KAFKA_HOST_KEY)); @@ -95,7 +95,7 @@ public void loadConfig(Map configs) { kafkaConfig.put("zookeeper.connect", configs.get(ZookeeperConfig.ZOOKEEPER_HOST_KEY) + ":" + configs.get(ZookeeperConfig.ZOOKEEPER_PORT_KEY)); } if (StringUtils.isNotEmpty(configs.get(ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY))) { - kafkaConfig.put("log.dirs", configs.get(ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY)); + kafkaConfig.put("log.dirs", getTmpDirPath(configs, ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY)); } if (StringUtils.isNotEmpty(configs.get(ConfluentConfig.CONFLUENT_KAFKA_BROKER_ID_KEY))) { kafkaConfig.put("broker.id", configs.get(ConfluentConfig.CONFLUENT_KAFKA_BROKER_ID_KEY)); @@ -152,7 +152,7 @@ public Bootstrap stop() { private void cleanup() { try { - FileUtils.deleteDirectory(Paths.get(configuration.getString(ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY)).toFile()); + FileUtils.deleteDirectory(Paths.get(getTmpDirPath(configuration, ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY)).toFile()); } catch (IOException e) { LOGGER.error("unable to delete {}", configuration.getString(ConfluentConfig.CONFLUENT_KAFKA_LOG_DIR_KEY), e); } diff --git a/hadoop-unit-confluent/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-confluent/src/main/resources/hadoop-unit-default.properties index 736fb25b..b55a6dfd 100644 --- a/hadoop-unit-confluent/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-confluent/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -99,9 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -133,19 +135,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase #knox.service=namenode,webhdfs,webhbase,oozie @@ -167,7 +169,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE @@ -181,7 +183,7 @@ confluent.schemaregistry.host=127.0.0.1 confluent.schemaregistry.kafkastore.topic=_schema confluent.schemaregistry.debug=false -confluent.kafka.log.dirs=/tmp/kafka-logs +confluent.kafka.log.dirs=/kafka-logs confluent.kafka.broker.id=0 confluent.kafka.port=22222 confluent.kafka.host=127.0.0.1 diff --git a/hadoop-unit-elasticsearch/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-elasticsearch/src/main/resources/hadoop-unit-default.properties index cce033c4..3c96bde6 100644 --- a/hadoop-unit-elasticsearch/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-elasticsearch/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,9 +90,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -111,7 +113,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -124,4 +126,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-elasticsearch/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-elasticsearch/src/test/resources/hadoop-unit-default.properties index f2505e16..3c96bde6 100644 --- a/hadoop-unit-elasticsearch/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-elasticsearch/src/test/resources/hadoop-unit-default.properties @@ -1,32 +1,20 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -41,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -56,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -74,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -102,8 +90,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -124,18 +113,17 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch -elasticsearch.version=6.7.1 +elasticsearch.version=5.4.3 elasticsearch.ip=127.0.0.1 elasticsearch.http.port=14433 elasticsearch.tcp.port=14533 elasticsearch.index.name=test_index elasticsearch.cluster.name=elasticsearch -#elasticsearch.download.url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.3.zip # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-hbase/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-hbase/src/main/resources/hadoop-unit-default.properties index 96d695c3..fb3a4870 100644 --- a/hadoop-unit-hbase/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-hbase/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -99,9 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -122,7 +124,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -135,4 +137,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-hbase/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-hbase/src/test/resources/hadoop-unit-default.properties index 5bf1600a..fb3a4870 100644 --- a/hadoop-unit-hbase/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-hbase/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -81,6 +83,9 @@ solr.collection.name=collection1 solr.cloud.port=8983 + + + # YARN yarn.num.node.managers=1 yarn.num.local.dirs=1 @@ -96,8 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -109,7 +115,26 @@ oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz oozie.port=20113 oozie.host=localhost +# MongoDB +mongo.ip=127.0.0.1 +mongo.port=13333 +mongo.database.name=test_database +mongo.collection.name=test_collection + +# Cassandra +cassandra.ip=127.0.0.1 +cassandra.port=13433 +cassandra.temp.dir=/embedded_cassandra + +# ElasticSearch +elasticsearch.version=5.4.3 +elasticsearch.ip=127.0.0.1 +elasticsearch.http.port=14433 +elasticsearch.tcp.port=14533 +elasticsearch.index.name=test_index +elasticsearch.cluster.name=elasticsearch + # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsBootstrap.java b/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsBootstrap.java index 95ce9f96..71815b5c 100644 --- a/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsBootstrap.java +++ b/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsBootstrap.java @@ -119,7 +119,7 @@ private void loadConfig() throws BootstrapException { host = configuration.getString(HdfsConfig.HDFS_NAMENODE_HOST_KEY); port = configuration.getInt(HdfsConfig.HDFS_NAMENODE_PORT_KEY); httpPort = configuration.getInt(HdfsConfig.HDFS_NAMENODE_HTTP_PORT_KEY); - tempDirectory = configuration.getString(HdfsConfig.HDFS_TEMP_DIR_KEY); + tempDirectory = getTmpDirPath(configuration, HdfsConfig.HDFS_TEMP_DIR_KEY); numDatanodes = configuration.getInt(HdfsConfig.HDFS_NUM_DATANODES_KEY); enablePermission = configuration.getBoolean(HdfsConfig.HDFS_ENABLE_PERMISSIONS_KEY); format = configuration.getBoolean(HdfsConfig.HDFS_FORMAT_KEY); @@ -143,7 +143,7 @@ public void loadConfig(Map configs) { httpPort = Integer.parseInt(configs.get(HdfsConfig.HDFS_NAMENODE_HTTP_PORT_KEY)); } if (StringUtils.isNotEmpty(configs.get(HdfsConfig.HDFS_TEMP_DIR_KEY))) { - tempDirectory = configs.get(HdfsConfig.HDFS_TEMP_DIR_KEY); + tempDirectory = getTmpDirPath(configs, HdfsConfig.HDFS_TEMP_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(HdfsConfig.HDFS_NUM_DATANODES_KEY))) { numDatanodes = Integer.parseInt(configs.get(HdfsConfig.HDFS_NUM_DATANODES_KEY)); diff --git a/hadoop-unit-hdfs/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-hdfs/src/main/resources/hadoop-unit-default.properties index 8ae7e8cf..2b55643c 100644 --- a/hadoop-unit-hdfs/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-hdfs/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -45,7 +47,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -63,7 +65,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,8 +93,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -107,4 +109,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-hdfs/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-hdfs/src/test/resources/hadoop-unit-default.properties index 219d8ff7..c0077bee 100644 --- a/hadoop-unit-hdfs/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-hdfs/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveMetastoreBootstrap.java b/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveMetastoreBootstrap.java index 2ec3a5fa..bfbbe3ed 100644 --- a/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveMetastoreBootstrap.java +++ b/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveMetastoreBootstrap.java @@ -77,8 +77,8 @@ public String getProperties() { private void loadConfig() throws BootstrapException { host = configuration.getString(HiveConfig.HIVE_METASTORE_HOSTNAME_KEY); port = configuration.getInt(HiveConfig.HIVE_METASTORE_PORT_KEY); - derbyDirectory = configuration.getString(HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); - scratchDirectory = configuration.getString(HiveConfig.HIVE_SCRATCH_DIR_KEY); + derbyDirectory = getTmpDirPath(configuration, HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); + scratchDirectory = getTmpDirPath(configuration, HiveConfig.HIVE_SCRATCH_DIR_KEY); warehouseDirectory = configuration.getString(HiveConfig.HIVE_WAREHOUSE_DIR_KEY); } @@ -91,10 +91,10 @@ public void loadConfig(Map configs) { port = Integer.parseInt(configs.get(HiveConfig.HIVE_METASTORE_PORT_KEY)); } if (StringUtils.isNotEmpty(configs.get(HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY))) { - derbyDirectory = configs.get(HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); + derbyDirectory = getTmpDirPath(configs, HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(HiveConfig.HIVE_SCRATCH_DIR_KEY))) { - scratchDirectory = configs.get(HiveConfig.HIVE_SCRATCH_DIR_KEY); + scratchDirectory = getTmpDirPath(configs, HiveConfig.HIVE_SCRATCH_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(HiveConfig.HIVE_WAREHOUSE_DIR_KEY))) { warehouseDirectory = configs.get(HiveConfig.HIVE_WAREHOUSE_DIR_KEY); diff --git a/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveServer2Bootstrap.java b/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveServer2Bootstrap.java index 1708f2a8..b317dcde 100644 --- a/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveServer2Bootstrap.java +++ b/hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveServer2Bootstrap.java @@ -87,8 +87,8 @@ private void loadConfig() throws BootstrapException { hostMetastore = configuration.getString(HiveConfig.HIVE_METASTORE_HOSTNAME_KEY); portMetastore = configuration.getInt(HiveConfig.HIVE_METASTORE_PORT_KEY); derbyDirectory = configuration.getString(HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); - scratchDirectory = configuration.getString(HiveConfig.HIVE_SCRATCH_DIR_KEY); - warehouseDirectory = configuration.getString(HiveConfig.HIVE_WAREHOUSE_DIR_KEY); + scratchDirectory = getTmpDirPath(configuration, HiveConfig.HIVE_SCRATCH_DIR_KEY); + warehouseDirectory = getTmpDirPath(configuration, HiveConfig.HIVE_WAREHOUSE_DIR_KEY); zookeeperConnectionString = configuration.getString(ZookeeperConfig.ZOOKEEPER_HOST_KEY) + ":" + configuration.getInt(ZookeeperConfig.ZOOKEEPER_PORT_KEY); hdfsUri = "hdfs://" + configuration.getString(HdfsConfig.HDFS_NAMENODE_HOST_KEY) + ":" + configuration.getString(HdfsConfig.HDFS_NAMENODE_PORT_KEY); } @@ -108,10 +108,10 @@ public void loadConfig(Map configs) { portMetastore = Integer.parseInt(configs.get(HiveConfig.HIVE_METASTORE_PORT_KEY)); } if (StringUtils.isNotEmpty(configs.get(HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY))) { - derbyDirectory = configs.get(HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); + derbyDirectory = getTmpDirPath(configs, HiveConfig.HIVE_METASTORE_DERBY_DB_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(HiveConfig.HIVE_SCRATCH_DIR_KEY))) { - scratchDirectory = configs.get(HiveConfig.HIVE_SCRATCH_DIR_KEY); + scratchDirectory = getTmpDirPath(configs, HiveConfig.HIVE_SCRATCH_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(HiveConfig.HIVE_WAREHOUSE_DIR_KEY))) { warehouseDirectory = configs.get(HiveConfig.HIVE_WAREHOUSE_DIR_KEY); diff --git a/hadoop-unit-hive/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-hive/src/main/resources/hadoop-unit-default.properties index 219d8ff7..c0077bee 100644 --- a/hadoop-unit-hive/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-hive/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-hive/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-hive/src/test/resources/hadoop-unit-default.properties index 258f0736..c0077bee 100644 --- a/hadoop-unit-hive/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-hive/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ - #HADOOP_HOME=/opt/hadoop +#HADOOP_HOME=/opt/hadoop + +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-kafka/src/main/java/fr/jetoile/hadoopunit/component/KafkaBootstrap.java b/hadoop-unit-kafka/src/main/java/fr/jetoile/hadoopunit/component/KafkaBootstrap.java index 21640c72..1a802de0 100644 --- a/hadoop-unit-kafka/src/main/java/fr/jetoile/hadoopunit/component/KafkaBootstrap.java +++ b/hadoop-unit-kafka/src/main/java/fr/jetoile/hadoopunit/component/KafkaBootstrap.java @@ -101,7 +101,7 @@ private void loadConfig() { host = configuration.getString(KafkaConfig.KAFKA_HOSTNAME_KEY); port = configuration.getInt(KafkaConfig.KAFKA_PORT_KEY); brokerId = configuration.getInt(KafkaConfig.KAFKA_TEST_BROKER_ID_KEY); - tmpDirectory = configuration.getString(KafkaConfig.KAFKA_TEST_TEMP_DIR_KEY); + tmpDirectory = getTmpDirPath(configuration, KafkaConfig.KAFKA_TEST_TEMP_DIR_KEY); zookeeperConnectionString = configuration.getString(ZookeeperConfig.ZOOKEEPER_HOST_KEY) + ":" + configuration.getInt(ZookeeperConfig.ZOOKEEPER_PORT_KEY); } @@ -117,7 +117,7 @@ public void loadConfig(Map configs) { brokerId = Integer.parseInt(configs.get(KafkaConfig.KAFKA_TEST_BROKER_ID_KEY)); } if (StringUtils.isNotEmpty(configs.get(KafkaConfig.KAFKA_TEST_TEMP_DIR_KEY))) { - tmpDirectory = configs.get(KafkaConfig.KAFKA_TEST_TEMP_DIR_KEY); + tmpDirectory = getTmpDirPath(configs, KafkaConfig.KAFKA_TEST_TEMP_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(ZookeeperConfig.ZOOKEEPER_HOST_KEY)) && StringUtils.isNotEmpty(configs.get(ZookeeperConfig.ZOOKEEPER_PORT_KEY))) { zookeeperConnectionString = configs.get(ZookeeperConfig.ZOOKEEPER_HOST_KEY) + ":" + configs.get(ZookeeperConfig.ZOOKEEPER_PORT_KEY); diff --git a/hadoop-unit-kafka/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-kafka/src/main/resources/hadoop-unit-default.properties index 219d8ff7..c0077bee 100644 --- a/hadoop-unit-kafka/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-kafka/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-kafka/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-kafka/src/test/resources/hadoop-unit-default.properties index 219d8ff7..c0077bee 100644 --- a/hadoop-unit-kafka/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-kafka/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-knox/src/main/java/fr/jetoile/hadoopunit/component/KnoxBootstrap.java b/hadoop-unit-knox/src/main/java/fr/jetoile/hadoopunit/component/KnoxBootstrap.java index 5c5a25f1..ede36c38 100644 --- a/hadoop-unit-knox/src/main/java/fr/jetoile/hadoopunit/component/KnoxBootstrap.java +++ b/hadoop-unit-knox/src/main/java/fr/jetoile/hadoopunit/component/KnoxBootstrap.java @@ -159,7 +159,7 @@ private void loadConfig() { host = configuration.getString(KnoxConfig.KNOX_HOST_KEY); path = configuration.getString(KnoxConfig.KNOX_PATH_KEY); clusterName = configuration.getString(KnoxConfig.KNOX_CLUSTER_KEY); - tempDirectory = configuration.getString(KnoxConfig.KNOX_HOME_DIR_KEY); + tempDirectory = getTmpDirPath(configuration, KnoxConfig.KNOX_HOME_DIR_KEY); List servicesList = Arrays.asList(configuration.getStringArray(KnoxConfig.KNOX_SERVICE_KEY)); services = Arrays.asList(KnoxService.values()).stream().filter(s -> servicesList.contains(s.getName())).collect(Collectors.toList()); @@ -186,7 +186,7 @@ public void loadConfig(Map configs) { clusterName = configs.get(KnoxConfig.KNOX_CLUSTER_KEY); } if (StringUtils.isNotEmpty(configs.get(KnoxConfig.KNOX_HOME_DIR_KEY))) { - tempDirectory = configs.get(KnoxConfig.KNOX_HOME_DIR_KEY); + tempDirectory = getTmpDirPath(configs, KnoxConfig.KNOX_HOME_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(KnoxConfig.KNOX_SERVICE_KEY))) { List servicesList = Arrays.asList(configuration.getStringArray(KnoxConfig.KNOX_SERVICE_KEY)); diff --git a/hadoop-unit-knox/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-knox/src/main/resources/hadoop-unit-default.properties index f01d913a..e2d85b9b 100644 --- a/hadoop-unit-knox/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-knox/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -96,8 +98,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -114,6 +116,6 @@ knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase,oozie #knox.service=namenode,webhdfs,webhbase diff --git a/hadoop-unit-maven-plugin/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-maven-plugin/src/main/resources/hadoop-unit-default.properties index f2ea062c..e3ed9466 100644 --- a/hadoop-unit-maven-plugin/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-maven-plugin/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -96,9 +98,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -129,19 +131,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase # Alluxio @@ -163,7 +165,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE diff --git a/hadoop-unit-mongodb/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-mongodb/src/main/resources/hadoop-unit-default.properties index 2391aa5a..42fcd366 100644 --- a/hadoop-unit-mongodb/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-mongodb/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,9 +93,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -114,7 +116,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -127,4 +129,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-mongodb/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-mongodb/src/test/resources/hadoop-unit-default.properties index e9a3d68d..64c1973c 100644 --- a/hadoop-unit-mongodb/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-mongodb/src/test/resources/hadoop-unit-default.properties @@ -14,19 +14,21 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -41,7 +43,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -56,7 +58,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -74,7 +76,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -102,8 +104,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -124,4 +126,4 @@ mongo.collection.name=test_collection # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-neo4j/src/main/java/fr/jetoile/hadoopunit/component/Neo4jBootstrap.java b/hadoop-unit-neo4j/src/main/java/fr/jetoile/hadoopunit/component/Neo4jBootstrap.java index 5df2b899..39ef0aa5 100644 --- a/hadoop-unit-neo4j/src/main/java/fr/jetoile/hadoopunit/component/Neo4jBootstrap.java +++ b/hadoop-unit-neo4j/src/main/java/fr/jetoile/hadoopunit/component/Neo4jBootstrap.java @@ -77,7 +77,7 @@ public String getProperties() { private void loadConfig() { port = configuration.getInt(Neo4jConfig.NEO4J_PORT_KEY); ip = configuration.getString(Neo4jConfig.NEO4J_IP_KEY); - tmp = configuration.getString(Neo4jConfig.NEO4J_TEMP_DIR_KEY); + tmp = getTmpDirPath(configuration, Neo4jConfig.NEO4J_TEMP_DIR_KEY); } @Override @@ -89,7 +89,7 @@ public void loadConfig(Map configs) { ip = configs.get(Neo4jConfig.NEO4J_IP_KEY); } if (StringUtils.isNotEmpty(configs.get(Neo4jConfig.NEO4J_TEMP_DIR_KEY))) { - tmp = configs.get(Neo4jConfig.NEO4J_TEMP_DIR_KEY); + tmp = getTmpDirPath(configs, Neo4jConfig.NEO4J_TEMP_DIR_KEY); } } @@ -143,9 +143,9 @@ public Bootstrap stop() { private void cleanup() { try { - FileUtils.deleteDirectory(Paths.get(configuration.getString(Neo4jConfig.NEO4J_TEMP_DIR_KEY)).toFile()); + FileUtils.deleteDirectory(Paths.get(tmp).toFile()); } catch (IOException e) { - LOGGER.error("unable to delete {}", configuration.getString(Neo4jConfig.NEO4J_TEMP_DIR_KEY), e); + LOGGER.error("unable to delete {}", tmp, e); } } diff --git a/hadoop-unit-neo4j/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-neo4j/src/main/resources/hadoop-unit-default.properties index 2391aa5a..42fcd366 100644 --- a/hadoop-unit-neo4j/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-neo4j/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,9 +93,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -114,7 +116,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -127,4 +129,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-neo4j/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-neo4j/src/test/resources/hadoop-unit-default.properties index fc0ba0d1..717e39a6 100644 --- a/hadoop-unit-neo4j/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-neo4j/src/test/resources/hadoop-unit-default.properties @@ -13,20 +13,21 @@ # #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -41,7 +42,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -56,7 +57,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -74,7 +75,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -102,8 +103,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -124,9 +125,9 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java b/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java index 56523c29..9194a63f 100644 --- a/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java +++ b/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java @@ -163,7 +163,7 @@ private void build() throws NotFoundServiceException { } private void loadConfig() throws BootstrapException, NotFoundServiceException { - oozieTestDir = configuration.getString(OozieConfig.OOZIE_TEST_DIR_KEY); + oozieTestDir = getTmpDirPath(configuration, OozieConfig.OOZIE_TEST_DIR_KEY); oozieHomeDir = configuration.getString(OozieConfig.OOZIE_HOME_DIR_KEY); oozieUsername = System.getProperty("user.name"); oozieGroupname = configuration.getString(OozieConfig.OOZIE_GROUPNAME_KEY); @@ -174,7 +174,7 @@ private void loadConfig() throws BootstrapException, NotFoundServiceException { oozieLocalShareLibCacheDir = configuration.getString(OozieConfig.OOZIE_LOCAL_SHARE_LIB_CACHE_DIR_KEY); ooziePurgeLocalShareLibCache = configuration.getBoolean(OozieConfig.OOZIE_PURGE_LOCAL_SHARE_LIB_CACHE_KEY); - oozieTmpDir = configuration.getString(OozieConfig.OOZIE_TMP_DIR_KEY); + oozieTmpDir = getTmpDirPath(configuration, OozieConfig.OOZIE_TMP_DIR_KEY); numNodeManagers = Integer.parseInt(configuration.getString(YarnConfig.YARN_NUM_NODE_MANAGERS_KEY)); jobHistoryAddress = configuration.getString(YarnConfig.MR_JOB_HISTORY_ADDRESS_KEY); @@ -198,7 +198,7 @@ private void loadConfig() throws BootstrapException, NotFoundServiceException { @Override public void loadConfig(Map configs) { if (StringUtils.isNotEmpty(configs.get(OozieConfig.OOZIE_TEST_DIR_KEY))) { - oozieTestDir = configs.get(OozieConfig.OOZIE_TEST_DIR_KEY); + oozieTestDir = getTmpDirPath(configs, OozieConfig.OOZIE_TEST_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(OozieConfig.OOZIE_HOME_DIR_KEY))) { oozieHomeDir = configs.get(OozieConfig.OOZIE_HOME_DIR_KEY); @@ -223,7 +223,7 @@ public void loadConfig(Map configs) { } if (StringUtils.isNotEmpty(configs.get(OozieConfig.OOZIE_TMP_DIR_KEY))) { - oozieTmpDir = configs.get(OozieConfig.OOZIE_TMP_DIR_KEY); + oozieTmpDir = getTmpDirPath(configs, OozieConfig.OOZIE_TMP_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(YarnConfig.YARN_NUM_NODE_MANAGERS_KEY))) { diff --git a/hadoop-unit-oozie/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-oozie/src/main/resources/hadoop-unit-default.properties index 68f76a24..4f3ef202 100644 --- a/hadoop-unit-oozie/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-oozie/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,9 +90,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -113,7 +115,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -126,4 +128,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-oozie/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-oozie/src/test/resources/hadoop-unit-default.properties index 676d894d..03e44da1 100644 --- a/hadoop-unit-oozie/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-oozie/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,9 +89,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -108,4 +109,4 @@ oozie.sharelib.component=OOZIE,MAPREDUCE_STREAMING # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-redis/src/main/java/fr/jetoile/hadoopunit/component/RedisBootstrap.java b/hadoop-unit-redis/src/main/java/fr/jetoile/hadoopunit/component/RedisBootstrap.java index c75f8ab8..27454887 100644 --- a/hadoop-unit-redis/src/main/java/fr/jetoile/hadoopunit/component/RedisBootstrap.java +++ b/hadoop-unit-redis/src/main/java/fr/jetoile/hadoopunit/component/RedisBootstrap.java @@ -111,7 +111,7 @@ private void loadConfig() { downloadUrl = configuration.getString(RedisConfig.REDIS_DOWNLOAD_URL_KEY); cleanupInstallation = configuration.getBoolean(RedisConfig.REDIS_CLEANUP_INSTALLATION_KEY); type = RedisType.valueOf(configuration.getString(RedisConfig.REDIS_TYPE_KEY, RedisType.SERVER.name())); - tmpDir = configuration.getString(RedisConfig.REDIS_TMP_DIR_KEY); + tmpDir = getTmpDirPath(configuration, RedisConfig.REDIS_TMP_DIR_KEY); if (configuration.containsKey(RedisConfig.REDIS_SLAVE_PORT_KEY)) { slavePorts = configuration.getList(RedisConfig.REDIS_SLAVE_PORT_KEY).stream().map(s -> Integer.valueOf(((String) s).trim())).collect(Collectors.toList()); @@ -142,7 +142,7 @@ public void loadConfig(Map configs) { type = RedisType.valueOf(configs.get(RedisConfig.REDIS_TYPE_KEY)); } if (StringUtils.isNotEmpty(configs.get(RedisConfig.REDIS_TMP_DIR_KEY))) { - tmpDir = configs.get(RedisConfig.REDIS_TMP_DIR_KEY); + tmpDir = getTmpDirPath(configs, RedisConfig.REDIS_TMP_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(RedisConfig.REDIS_SLAVE_PORT_KEY))) { List ports = Arrays.asList(configs.get(RedisConfig.REDIS_SLAVE_PORT_KEY).split(",")); diff --git a/hadoop-unit-redis/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-redis/src/main/resources/hadoop-unit-default.properties index 46ae0cec..014f445d 100644 --- a/hadoop-unit-redis/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-redis/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,9 +90,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -113,7 +115,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -126,7 +128,7 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # Redis @@ -134,7 +136,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE diff --git a/hadoop-unit-redis/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-redis/src/test/resources/hadoop-unit-default.properties index 5fe48bde..ea88086a 100644 --- a/hadoop-unit-redis/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-redis/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,9 +89,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -113,7 +114,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -126,7 +127,7 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # Redis @@ -134,7 +135,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=5.0.4 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE diff --git a/hadoop-unit-solr/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-solr/src/main/resources/hadoop-unit-default.properties index 52cbb260..a2a67b70 100644 --- a/hadoop-unit-solr/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-solr/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,9 +92,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -114,9 +115,9 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-solr/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-solr/src/test/resources/hadoop-unit-default.properties index 219d8ff7..1b328f75 100644 --- a/hadoop-unit-solr/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-solr/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +89,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +105,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-solrcloud/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-solrcloud/src/main/resources/hadoop-unit-default.properties index 219d8ff7..1b328f75 100644 --- a/hadoop-unit-solrcloud/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-solrcloud/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +89,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +105,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-solrcloud/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-solrcloud/src/test/resources/hadoop-unit-default.properties index 219d8ff7..1b328f75 100644 --- a/hadoop-unit-solrcloud/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-solrcloud/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +89,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +105,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties index 80d8aca0..03158a83 100644 --- a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties @@ -1,5 +1,7 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + zookeeper.artifactId=fr.jetoile.hadoop:hadoop-unit-zookeeper:${project.version} zookeeper.mainClass=fr.jetoile.hadoopunit.component.ZookeeperBootstrap zookeeper.metadataClass=fr.jetoile.hadoopunit.component.ZookeeperMetadata @@ -114,18 +116,18 @@ maven.local.repo=~/.m2/repository maven.debug=true # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=127.0.0.1 hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=127.0.0.1 @@ -140,7 +142,7 @@ hive.test.table.name=test_table hdfs.namenode.host=127.0.0.1 hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -158,7 +160,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -184,7 +186,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -215,9 +217,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=127.0.0.1:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -249,19 +251,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=127.0.0.1 knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase #knox.service=namenode,webhdfs,webhbase,oozie @@ -283,7 +285,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=5.0.4 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE @@ -300,7 +302,7 @@ confluent.schemaregistry.host=127.0.0.1 confluent.schemaregistry.kafkastore.topic=_schema confluent.schemaregistry.debug=false -confluent.kafka.log.dirs=/tmp/kafka-logs +confluent.kafka.log.dirs=/kafka-logs confluent.kafka.broker.id=0 confluent.kafka.port=22222 confluent.kafka.host=127.0.0.1 diff --git a/hadoop-unit-yarn/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-yarn/src/main/resources/hadoop-unit-default.properties index bfe01ecd..29fd49f9 100644 --- a/hadoop-unit-yarn/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-yarn/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -99,9 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -133,19 +135,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase # Alluxio @@ -166,7 +168,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE diff --git a/hadoop-unit-yarn/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-yarn/src/test/resources/hadoop-unit-default.properties index bfe01ecd..29fd49f9 100644 --- a/hadoop-unit-yarn/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-yarn/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -99,9 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -133,19 +135,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase # Alluxio @@ -166,7 +168,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE diff --git a/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java b/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java index 0c035291..d8d95c58 100644 --- a/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java +++ b/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java @@ -79,7 +79,7 @@ public String getProperties() { private void loadConfig() throws BootstrapException { port = configuration.getInt(ZookeeperConfig.ZOOKEEPER_PORT_KEY); - localDir = configuration.getString(ZookeeperConfig.ZOOKEEPER_TEMP_DIR_KEY); + localDir = getTmpDirPath(configuration, ZookeeperConfig.ZOOKEEPER_TEMP_DIR_KEY); host = configuration.getString(ZookeeperConfig.ZOOKEEPER_HOST_KEY); } @@ -90,7 +90,7 @@ public void loadConfig(Map configs) { port = Integer.parseInt(configs.get(ZookeeperConfig.ZOOKEEPER_PORT_KEY)); } if (StringUtils.isNotEmpty(configs.get(ZookeeperConfig.ZOOKEEPER_TEMP_DIR_KEY))) { - localDir = configs.get(ZookeeperConfig.ZOOKEEPER_TEMP_DIR_KEY); + localDir = getTmpDirPath(configs, ZookeeperConfig.ZOOKEEPER_TEMP_DIR_KEY); } if (StringUtils.isNotEmpty(configs.get(ZookeeperConfig.ZOOKEEPER_HOST_KEY))) { host = configs.get(ZookeeperConfig.ZOOKEEPER_HOST_KEY); diff --git a/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties index 219d8ff7..c0077bee 100644 --- a/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/hadoop-unit-zookeeper/src/test/resources/hadoop-unit-default.properties b/hadoop-unit-zookeeper/src/test/resources/hadoop-unit-default.properties index 219d8ff7..c0077bee 100644 --- a/hadoop-unit-zookeeper/src/test/resources/hadoop-unit-default.properties +++ b/hadoop-unit-zookeeper/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -104,4 +106,4 @@ oozie.host=localhost # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/sample/all/src/test/resources/hadoop-unit-default.properties b/sample/all/src/test/resources/hadoop-unit-default.properties index c5854fbc..f0b2ec8c 100644 --- a/sample/all/src/test/resources/hadoop-unit-default.properties +++ b/sample/all/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -41,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -67,7 +69,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -95,9 +97,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -118,9 +120,9 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/sample/confluent-integrationtest/src/test/resources/hadoop-unit-default.properties b/sample/confluent-integrationtest/src/test/resources/hadoop-unit-default.properties index f3e37a1a..5f4a8e75 100644 --- a/sample/confluent-integrationtest/src/test/resources/hadoop-unit-default.properties +++ b/sample/confluent-integrationtest/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -68,7 +70,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -99,9 +101,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -133,19 +135,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase #knox.service=namenode,webhdfs,webhbase,oozie @@ -167,7 +169,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE @@ -184,7 +186,7 @@ confluent.schemaregistry.host=127.0.0.1 confluent.schemaregistry.kafkastore.topic=_schema confluent.schemaregistry.debug=false -confluent.kafka.log.dirs=/tmp/kafka-logs +confluent.kafka.log.dirs=/kafka-logs confluent.kafka.broker.id=0 confluent.kafka.port=22222 confluent.kafka.host=127.0.0.1 diff --git a/sample/kafka-spark-streaming/src/test/resources/hadoop-unit-default.properties b/sample/kafka-spark-streaming/src/test/resources/hadoop-unit-default.properties index 21eaf9b5..469e0a07 100644 --- a/sample/kafka-spark-streaming/src/test/resources/hadoop-unit-default.properties +++ b/sample/kafka-spark-streaming/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,8 +93,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -113,7 +115,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -126,4 +128,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/sample/kafka-stream/src/test/resources/hadoop-unit-default.properties b/sample/kafka-stream/src/test/resources/hadoop-unit-default.properties index 87a5e877..875ec2a2 100644 --- a/sample/kafka-stream/src/test/resources/hadoop-unit-default.properties +++ b/sample/kafka-stream/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -110,9 +112,9 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/sample/knox-hbase-webhdfs/src/test/resources/hadoop-unit-default.properties b/sample/knox-hbase-webhdfs/src/test/resources/hadoop-unit-default.properties index 0308ec2a..83901b08 100644 --- a/sample/knox-hbase-webhdfs/src/test/resources/hadoop-unit-default.properties +++ b/sample/knox-hbase-webhdfs/src/test/resources/hadoop-unit-default.properties @@ -1,17 +1,19 @@ +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -26,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -41,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -67,7 +69,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -98,9 +100,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -129,19 +131,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase # Alluxio diff --git a/sample/parquet-spark/src/test/resources/hadoop-unit-default.properties b/sample/parquet-spark/src/test/resources/hadoop-unit-default.properties index 87a5e877..875ec2a2 100644 --- a/sample/parquet-spark/src/test/resources/hadoop-unit-default.properties +++ b/sample/parquet-spark/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -110,9 +112,9 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/sample/redis/src/test/resources/hadoop-unit-default.properties b/sample/redis/src/test/resources/hadoop-unit-default.properties index e9c85368..232f0019 100644 --- a/sample/redis/src/test/resources/hadoop-unit-default.properties +++ b/sample/redis/src/test/resources/hadoop-unit-default.properties @@ -1,5 +1,7 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + zookeeper.artifactId=fr.jetoile.hadoop:hadoop-unit-zookeeper:2.3-SNAPSHOT hdfs.artifactId=fr.jetoile.hadoop:hadoop-unit-hdfs:2.3-SNAPSHOT alluxio.artifactId=fr.jetoile.hadoop:hadoop-unit-alluxio:2.3-SNAPSHOT @@ -21,18 +23,18 @@ maven.central.repo=https://repo.maven.apache.org/maven2/ maven.local.repo=/home/khanh/.m2/repository # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -47,7 +49,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -62,7 +64,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -88,7 +90,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -119,9 +121,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -153,19 +155,19 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j +neo4j.temp.dir=/embedded_neo4j # KNOX knox.host=localhost knox.port=8888 knox.path=gateway knox.cluster=mycluster -knox.home.dir=/tmp/embedded_knox +knox.home.dir=/embedded_knox knox.service=namenode,webhdfs,webhbase # Alluxio @@ -186,7 +188,7 @@ redis.port=6379 redis.download.url=http://download.redis.io/releases/ redis.version=4.0.0 redis.cleanup.installation=false -redis.temp.dir=/tmp/redis +redis.temp.dir=/redis redis.type=SERVER #redis.type=CLUSTER #redis.type=MASTER_SLAVE diff --git a/sample/solr-parquet-spark/src/test/resources/hadoop-unit-default.properties b/sample/solr-parquet-spark/src/test/resources/hadoop-unit-default.properties index 87a5e877..875ec2a2 100644 --- a/sample/solr-parquet-spark/src/test/resources/hadoop-unit-default.properties +++ b/sample/solr-parquet-spark/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,20 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp + # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +29,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +44,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +62,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -88,8 +90,8 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -110,9 +112,9 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file diff --git a/sample/spark-streaming-cassandra/src/test/resources/hadoop-unit-default.properties b/sample/spark-streaming-cassandra/src/test/resources/hadoop-unit-default.properties index 2391aa5a..44dc09c7 100644 --- a/sample/spark-streaming-cassandra/src/test/resources/hadoop-unit-default.properties +++ b/sample/spark-streaming-cassandra/src/test/resources/hadoop-unit-default.properties @@ -1,18 +1,19 @@ #HADOOP_HOME=/opt/hadoop +tmp.dir.path=/tmp # Zookeeper -zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.temp.dir=/embedded_zk zookeeper.host=127.0.0.1 zookeeper.port=22010 # Hive -hive.scratch.dir=/tmp/hive_scratch_dir +hive.scratch.dir=/hive_scratch_dir hive.warehouse.dir=/tmp/warehouse_dir # Hive Metastore hive.metastore.hostname=localhost hive.metastore.port=20102 -hive.metastore.derby.db.dir=metastore_db +hive.metastore.derby.db.dir=/metastore_db # Hive Server2 hive.server2.hostname=localhost @@ -27,7 +28,7 @@ hive.test.table.name=test_table hdfs.namenode.host=localhost hdfs.namenode.port=20112 hdfs.namenode.http.port=50070 -hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.temp.dir=/embedded_hdfs hdfs.num.datanodes=1 hdfs.enable.permissions=false hdfs.format=true @@ -42,7 +43,7 @@ hdfs.test.string=TESTING hbase.master.port=25111 hbase.master.info.port=-1 hbase.num.region.servers=1 -hbase.root.dir=/tmp/embedded_hbase +hbase.root.dir=/embedded_hbase hbase.znode.parent=/hbase-unsecure hbase.wal.replication.enabled=false @@ -60,7 +61,7 @@ kafka.port=20111 kafka.test.topic=testtopic kafka.test.message.count=10 kafka.test.broker.id=1 -kafka.test.temp.dir=embedded_kafka +kafka.test.temp.dir=/embedded_kafka #SolR + SolRCloud solr.dir=solr @@ -91,9 +92,9 @@ yarn.use.in.jvm.container.executor=false mr.job.history.address=localhost:37005 # Oozie -oozie.tmp.dir=/tmp/oozie_tmp -oozie.test.dir=/tmp/embedded_oozie -oozie.home.dir=/tmp/oozie_home +oozie.tmp.dir=/oozie_tmp +oozie.test.dir=/embedded_oozie +oozie.home.dir=/oozie_home oozie.username=blah oozie.groupname=testgroup oozie.hdfs.share.lib.dir=/tmp/share_lib @@ -114,7 +115,7 @@ mongo.collection.name=test_collection # Cassandra cassandra.ip=127.0.0.1 cassandra.port=13433 -cassandra.temp.dir=/tmp/embedded_cassandra +cassandra.temp.dir=/embedded_cassandra # ElasticSearch elasticsearch.version=5.4.3 @@ -127,4 +128,4 @@ elasticsearch.cluster.name=elasticsearch # Neo4j neo4j.ip=127.0.0.1 neo4j.port=13533 -neo4j.temp.dir=/tmp/embedded_neo4j \ No newline at end of file +neo4j.temp.dir=/embedded_neo4j \ No newline at end of file