From 419156e09eb0638aa926cd7c8fbb3dcefc3efd1b Mon Sep 17 00:00:00 2001 From: jetoile Date: Sat, 21 Jan 2017 18:52:31 +0100 Subject: [PATCH] v2.1 --- README.md | 28 +++++++++-------- hadoop-unit-cassandra/pom.xml | 2 +- .../hadoop-unit-client-hdfs/pom.xml | 2 +- .../hadoop-unit-client-hive/pom.xml | 2 +- .../hadoop-unit-client-kafka/pom.xml | 2 +- .../hadoop-unit-client-solrcloud/pom.xml | 2 +- .../hadoop-unit-client-spark/pom.xml | 2 +- hadoop-unit-client/pom.xml | 2 +- hadoop-unit-commons/pom.xml | 2 +- hadoop-unit-elasticsearch/pom.xml | 2 +- hadoop-unit-hbase/pom.xml | 2 +- hadoop-unit-hdfs/pom.xml | 2 +- hadoop-unit-hive/pom.xml | 2 +- hadoop-unit-kafka/pom.xml | 2 +- hadoop-unit-knox/pom.xml | 2 +- hadoop-unit-maven-plugin/pom.xml | 2 +- hadoop-unit-mongodb/pom.xml | 2 +- hadoop-unit-neo4j/pom.xml | 2 +- hadoop-unit-oozie/pom.xml | 2 +- hadoop-unit-solr/pom.xml | 2 +- hadoop-unit-solrcloud/pom.xml | 2 +- hadoop-unit-standalone/pom.xml | 2 +- .../resources/hadoop-unit-default.properties | 30 +++++++++---------- .../ManualIntegrationBootstrapTest.java | 6 ++-- hadoop-unit-zookeeper/pom.xml | 2 +- pom.xml | 2 +- sample/all/pom.xml | 2 +- sample/kafka-spark-streaming/pom.xml | 2 +- sample/kafka-stream/pom.xml | 2 +- sample/parquet-spark/pom.xml | 2 +- sample/pom.xml | 2 +- sample/solr-parquet-spark/pom.xml | 2 +- sample/spark-streaming-cassandra/pom.xml | 2 +- 33 files changed, 64 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 45dadb9f..3b66fd6f 100755 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ mvn package | Hadoop Unit version | Hadoop mini cluster version | HDP version | | ------------- | ------------- | ------------- | +| 2.1 | 0.1.11 | HDP 2.5.3.0 | | 2.0 | 0.1.9 | HDP 2.5.3.0 | | 1.5 | 0.1.8 | HDP 2.5.0.0 | | 1.4 | 0.1.7 | HDP 2.4.2.0 | @@ -66,6 +67,7 @@ The available components are: * CASSANDRA * ELASTICSEARCH * NEO4J +* KNOX ##Integration testing (will start each component present into classpath) With maven, add dependencies of components which are needed @@ -75,7 +77,7 @@ Sample: fr.jetoile.hadoop hadoop-unit-hdfs - 2.0 + 2.1 test ``` @@ -101,7 +103,7 @@ Sample: fr.jetoile.hadoop hadoop-unit-hdfs - 2.0 + 2.1 test ``` @@ -238,21 +240,21 @@ To use it, add into the pom project stuff like that: fr.jetoile.hadoop hadoop-unit-client-hdfs - 2.0 + 2.1 test fr.jetoile.hadoop hadoop-unit-client-hive - 2.0 + 2.1 test fr.jetoile.hadoop hadoop-unit-client-spark - 2.0 + 2.1 test @@ -303,19 +305,19 @@ To use it, add into the pom project stuff like that: HDFS - fr.jetoile.hadoop:hadoop-unit-hdfs:2.0 + fr.jetoile.hadoop:hadoop-unit-hdfs:2.1 ZOOKEEPER - fr.jetoile.hadoop:hadoop-unit-zookeeper:2.0 + fr.jetoile.hadoop:hadoop-unit-zookeeper:2.1 HIVEMETA - fr.jetoile.hadoop:hadoop-unit-hive:2.0 + fr.jetoile.hadoop:hadoop-unit-hive:2.1 HIVESERVER2 - fr.jetoile.hadoop:hadoop-unit-hive:2.0 + fr.jetoile.hadoop:hadoop-unit-hive:2.1 SOLRCLOUD @@ -347,6 +349,7 @@ Values can be: * CASSANDRA * ELASTICSEARCH * NEO4J +* KNOX It is also possible to override configurations with a list of `properties` which accept a map (ie. `value` and where `key` is a property from the file `hadoop-unit-default.properties`). @@ -443,7 +446,7 @@ To use it, add into the pom project stuff like that: hadoop-unit-maven-plugin fr.jetoile.hadoop - 2.0 + 2.1 start @@ -470,7 +473,7 @@ To use it, add into the pom project stuff like that: hadoop-unit-maven-plugin fr.jetoile.hadoop - 2.0 + 2.1 stop @@ -570,9 +573,10 @@ public class HdfsBootstrapIntegrationTest { * Cassandra 3.4 * ElasticSearch 5.0-alpha4 * Neo4j 3.0.3 +* Knox Built on: -* [hadoop-mini-cluster-0.1.9](https://github.com/sakserv/hadoop-mini-clusters) (aka. HDP 2.5.3.0) +* [hadoop-mini-cluster-0.1.11](https://github.com/sakserv/hadoop-mini-clusters) (aka. HDP 2.5.3.0) * [achilles-embedded-4.2.0](https://github.com/doanduyhai/Achilles) * [maven aether](https://github.com/apache/maven-resolver/) diff --git a/hadoop-unit-cassandra/pom.xml b/hadoop-unit-cassandra/pom.xml index b3d53a44..0f812df8 100644 --- a/hadoop-unit-cassandra/pom.xml +++ b/hadoop-unit-cassandra/pom.xml @@ -6,7 +6,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-client/hadoop-unit-client-hdfs/pom.xml b/hadoop-unit-client/hadoop-unit-client-hdfs/pom.xml index 0ebfcaff..9ddd0f16 100755 --- a/hadoop-unit-client/hadoop-unit-client-hdfs/pom.xml +++ b/hadoop-unit-client/hadoop-unit-client-hdfs/pom.xml @@ -4,7 +4,7 @@ hadoop-unit-client fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-client/hadoop-unit-client-hive/pom.xml b/hadoop-unit-client/hadoop-unit-client-hive/pom.xml index 2f0dfb5d..ffa460ca 100755 --- a/hadoop-unit-client/hadoop-unit-client-hive/pom.xml +++ b/hadoop-unit-client/hadoop-unit-client-hive/pom.xml @@ -4,7 +4,7 @@ hadoop-unit-client fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-client/hadoop-unit-client-kafka/pom.xml b/hadoop-unit-client/hadoop-unit-client-kafka/pom.xml index b3923631..61d0b43c 100644 --- a/hadoop-unit-client/hadoop-unit-client-kafka/pom.xml +++ b/hadoop-unit-client/hadoop-unit-client-kafka/pom.xml @@ -5,7 +5,7 @@ hadoop-unit-client fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-client/hadoop-unit-client-solrcloud/pom.xml b/hadoop-unit-client/hadoop-unit-client-solrcloud/pom.xml index 738ecb99..97a78c95 100644 --- a/hadoop-unit-client/hadoop-unit-client-solrcloud/pom.xml +++ b/hadoop-unit-client/hadoop-unit-client-solrcloud/pom.xml @@ -5,7 +5,7 @@ hadoop-unit-client fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-client/hadoop-unit-client-spark/pom.xml b/hadoop-unit-client/hadoop-unit-client-spark/pom.xml index 9fec9ab1..256e5e27 100644 --- a/hadoop-unit-client/hadoop-unit-client-spark/pom.xml +++ b/hadoop-unit-client/hadoop-unit-client-spark/pom.xml @@ -5,7 +5,7 @@ hadoop-unit-client fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-client/pom.xml b/hadoop-unit-client/pom.xml index 03e3ec70..25220e6f 100755 --- a/hadoop-unit-client/pom.xml +++ b/hadoop-unit-client/pom.xml @@ -4,7 +4,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-commons/pom.xml b/hadoop-unit-commons/pom.xml index 1f642cd2..22545038 100644 --- a/hadoop-unit-commons/pom.xml +++ b/hadoop-unit-commons/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-elasticsearch/pom.xml b/hadoop-unit-elasticsearch/pom.xml index b959bf3e..ff29a225 100644 --- a/hadoop-unit-elasticsearch/pom.xml +++ b/hadoop-unit-elasticsearch/pom.xml @@ -6,7 +6,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-hbase/pom.xml b/hadoop-unit-hbase/pom.xml index 1afaabf7..61e27413 100644 --- a/hadoop-unit-hbase/pom.xml +++ b/hadoop-unit-hbase/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-hdfs/pom.xml b/hadoop-unit-hdfs/pom.xml index b2df51ba..fbff48e9 100644 --- a/hadoop-unit-hdfs/pom.xml +++ b/hadoop-unit-hdfs/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-hive/pom.xml b/hadoop-unit-hive/pom.xml index d7318e81..a4194073 100644 --- a/hadoop-unit-hive/pom.xml +++ b/hadoop-unit-hive/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-kafka/pom.xml b/hadoop-unit-kafka/pom.xml index d1fb8b55..4978fc02 100644 --- a/hadoop-unit-kafka/pom.xml +++ b/hadoop-unit-kafka/pom.xml @@ -19,7 +19,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-knox/pom.xml b/hadoop-unit-knox/pom.xml index c960a2f6..4891c807 100644 --- a/hadoop-unit-knox/pom.xml +++ b/hadoop-unit-knox/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-maven-plugin/pom.xml b/hadoop-unit-maven-plugin/pom.xml index 19cb8f48..8ddd3e71 100644 --- a/hadoop-unit-maven-plugin/pom.xml +++ b/hadoop-unit-maven-plugin/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-mongodb/pom.xml b/hadoop-unit-mongodb/pom.xml index 5a3555d3..1ee2097b 100644 --- a/hadoop-unit-mongodb/pom.xml +++ b/hadoop-unit-mongodb/pom.xml @@ -6,7 +6,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-neo4j/pom.xml b/hadoop-unit-neo4j/pom.xml index 2fe69896..4cead0e0 100644 --- a/hadoop-unit-neo4j/pom.xml +++ b/hadoop-unit-neo4j/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-oozie/pom.xml b/hadoop-unit-oozie/pom.xml index 0626e130..15d634f8 100644 --- a/hadoop-unit-oozie/pom.xml +++ b/hadoop-unit-oozie/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-solr/pom.xml b/hadoop-unit-solr/pom.xml index 7d9e5873..e5d99fa3 100644 --- a/hadoop-unit-solr/pom.xml +++ b/hadoop-unit-solr/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-solrcloud/pom.xml b/hadoop-unit-solrcloud/pom.xml index 61381cf2..51ef1e38 100644 --- a/hadoop-unit-solrcloud/pom.xml +++ b/hadoop-unit-solrcloud/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-standalone/pom.xml b/hadoop-unit-standalone/pom.xml index a94830f0..9d2ad2a3 100644 --- a/hadoop-unit-standalone/pom.xml +++ b/hadoop-unit-standalone/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties index 4371cb55..be99f3a0 100644 --- a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties @@ -1,19 +1,19 @@ #HADOOP_HOME=/opt/hadoop -zookeeper.artifact=fr.jetoile.hadoop:hadoop-unit-zookeeper:2.1-SNAPSHOT -hdfs.artifact=fr.jetoile.hadoop:hadoop-unit-hdfs:2.1-SNAPSHOT -hivemeta.artifact=fr.jetoile.hadoop:hadoop-unit-hive:2.1-SNAPSHOT -hiveserver2.artifact=fr.jetoile.hadoop:hadoop-unit-hive:2.1-SNAPSHOT -kafka.artifact=fr.jetoile.hadoop:hadoop-unit-kafka:2.1-SNAPSHOT -hbase.artifact=fr.jetoile.hadoop:hadoop-unit-hbase:2.1-SNAPSHOT -oozie.artifact=fr.jetoile.hadoop:hadoop-unit-oozie:2.1-SNAPSHOT -solr.artifact=fr.jetoile.hadoop:hadoop-unit-solr:2.1-SNAPSHOT -solrcloud.artifact=fr.jetoile.hadoop:hadoop-unit-solrcloud:2.1-SNAPSHOT -cassandra.artifact=fr.jetoile.hadoop:hadoop-unit-cassandra:2.1-SNAPSHOT -mongodb.artifact=fr.jetoile.hadoop:hadoop-unit-mongodb:2.1-SNAPSHOT -elasticsearch.artifact=fr.jetoile.hadoop:hadoop-unit-elasticsearch:2.1-SNAPSHOT -neo4j.artifact=fr.jetoile.hadoop:hadoop-unit-neo4j:2.1-SNAPSHOT -knox.artifact=fr.jetoile.hadoop:hadoop-unit-knox:2.1-SNAPSHOT +zookeeper.artifact=fr.jetoile.hadoop:hadoop-unit-zookeeper:2.1 +hdfs.artifact=fr.jetoile.hadoop:hadoop-unit-hdfs:2.1 +hivemeta.artifact=fr.jetoile.hadoop:hadoop-unit-hive:2.1 +hiveserver2.artifact=fr.jetoile.hadoop:hadoop-unit-hive:2.1 +kafka.artifact=fr.jetoile.hadoop:hadoop-unit-kafka:2.1 +hbase.artifact=fr.jetoile.hadoop:hadoop-unit-hbase:2.1 +oozie.artifact=fr.jetoile.hadoop:hadoop-unit-oozie:2.1 +solr.artifact=fr.jetoile.hadoop:hadoop-unit-solr:2.1 +solrcloud.artifact=fr.jetoile.hadoop:hadoop-unit-solrcloud:2.1 +cassandra.artifact=fr.jetoile.hadoop:hadoop-unit-cassandra:2.1 +mongodb.artifact=fr.jetoile.hadoop:hadoop-unit-mongodb:2.1 +elasticsearch.artifact=fr.jetoile.hadoop:hadoop-unit-elasticsearch:2.1 +neo4j.artifact=fr.jetoile.hadoop:hadoop-unit-neo4j:2.1 +knox.artifact=fr.jetoile.hadoop:hadoop-unit-knox:2.1 maven.central.repo=https://repo.maven.apache.org/maven2/ maven.local.repo=/home/khanh/.m2/repository @@ -127,7 +127,7 @@ oozie.share.lib.create=true oozie.local.share.lib.cache.dir=/tmp/share_lib_cache oozie.purge.local.share.lib.cache=false oozie.sharelib.path=/home/khanh/github -oozie.sharelib.name=oozie-4.2.1-SNAPSHOT.2.3.2.1-SNAPSHOT-2950-distro.tar.gz +oozie.sharelib.name=oozie-4.2.1.2.3.2.1-2950-distro.tar.gz oozie.port=20113 oozie.host=localhost diff --git a/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java b/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java index 0ead90ff..83ba443a 100644 --- a/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java +++ b/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java @@ -245,7 +245,7 @@ public void hdfsShouldStart() throws Exception { URL url = new URL( String.format("http://%s:%s/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest", - configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_HOST_KEY), + configuration.getString(HadoopUnitConfig.HDFS_NAMENODE_HOST_KEY), configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_HTTP_PORT_KEY))); URLConnection connection = url.openConnection(); connection.setRequestProperty("Accept-Charset", "UTF-8"); @@ -296,9 +296,9 @@ public void oozieShouldStart() throws Exception { LOGGER.info("OOZIE: Test Submit Workflow Start"); org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); - conf.set("fs.default.name", "hdfs://" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_HOST_KEY) + ":" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY)); + conf.set("fs.default.name", "hdfs://" + configuration.getString(HadoopUnitConfig.HDFS_NAMENODE_HOST_KEY) + ":" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY)); - URI uri = URI.create("hdfs://" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_HOST_KEY) + ":" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY)); + URI uri = URI.create("hdfs://" + configuration.getString(HadoopUnitConfig.HDFS_NAMENODE_HOST_KEY) + ":" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY)); FileSystem hdfsFs = FileSystem.get(uri, conf); diff --git a/hadoop-unit-zookeeper/pom.xml b/hadoop-unit-zookeeper/pom.xml index b89e351e..a23e352d 100644 --- a/hadoop-unit-zookeeper/pom.xml +++ b/hadoop-unit-zookeeper/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/pom.xml b/pom.xml index cae5da21..63c3ed92 100755 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ fr.jetoile.hadoop hadoop-unit pom - 2.1-SNAPSHOT + 2.1 hadoop-unit https://github.com/jetoile/hadoop-unit diff --git a/sample/all/pom.xml b/sample/all/pom.xml index 465ff5e2..5746fee0 100644 --- a/sample/all/pom.xml +++ b/sample/all/pom.xml @@ -5,7 +5,7 @@ sample fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/sample/kafka-spark-streaming/pom.xml b/sample/kafka-spark-streaming/pom.xml index 167ccbfd..49828a2c 100644 --- a/sample/kafka-spark-streaming/pom.xml +++ b/sample/kafka-spark-streaming/pom.xml @@ -5,7 +5,7 @@ sample fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/sample/kafka-stream/pom.xml b/sample/kafka-stream/pom.xml index db2692b0..afbc7072 100644 --- a/sample/kafka-stream/pom.xml +++ b/sample/kafka-stream/pom.xml @@ -5,7 +5,7 @@ sample fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/sample/parquet-spark/pom.xml b/sample/parquet-spark/pom.xml index 4ec10318..accba673 100644 --- a/sample/parquet-spark/pom.xml +++ b/sample/parquet-spark/pom.xml @@ -5,7 +5,7 @@ sample fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/sample/pom.xml b/sample/pom.xml index a7895b67..22cd7378 100644 --- a/sample/pom.xml +++ b/sample/pom.xml @@ -5,7 +5,7 @@ hadoop-unit fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 pom diff --git a/sample/solr-parquet-spark/pom.xml b/sample/solr-parquet-spark/pom.xml index 0abf22cf..612e613e 100644 --- a/sample/solr-parquet-spark/pom.xml +++ b/sample/solr-parquet-spark/pom.xml @@ -5,7 +5,7 @@ sample fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0 diff --git a/sample/spark-streaming-cassandra/pom.xml b/sample/spark-streaming-cassandra/pom.xml index 252d62f5..6aae228d 100644 --- a/sample/spark-streaming-cassandra/pom.xml +++ b/sample/spark-streaming-cassandra/pom.xml @@ -5,7 +5,7 @@ sample fr.jetoile.hadoop - 2.1-SNAPSHOT + 2.1 4.0.0