use aether to manage jar conflict

jetoile · Jan 5, 2017 · 07bfc41 · 07bfc41
1 parent a371998
commit 07bfc41
Show file tree

Hide file tree

Showing 131 changed files with 6,731 additions and 2,012 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,5 +10,6 @@ target
 *.log
 *.log.gz
 rebel.xml
-metastore_db
+**/metastore_db
+**/embedded_kafka
 **/dependency-reduced-pom.xml
diff --git a/hadoop-unit-cassandra/pom.xml b/hadoop-unit-cassandra/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <artifactId>hadoop-unit</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -23,21 +23,6 @@
             <groupId>info.archinnov</groupId>
             <artifactId>achilles-embedded</artifactId>
         </dependency>
-
-        <!--<dependency>-->
-        <!--<groupId>org.cassandraunit</groupId>-->
-        <!--<artifactId>cassandra-unit</artifactId>-->
-        <!--</dependency>-->
-
-        <!--<dependency>-->
-        <!--<groupId>com.datastax.cassandra</groupId>-->
-        <!--<artifactId>cassandra-driver-core</artifactId>-->
-        <!--</dependency>-->
-
-        <!--<dependency>-->
-        <!--<groupId>org.yaml</groupId>-->
-        <!--<artifactId>snakeyaml</artifactId>-->
-        <!--</dependency>-->
     </dependencies>
 
 </project>
diff --git a/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java b/hadoop-unit-cassandra/src/main/java/fr/jetoile/hadoopunit/component/CassandraBootstrap.java
@@ -16,28 +16,29 @@
 
 import com.datastax.driver.core.Session;
 import fr.jetoile.hadoopunit.Component;
+import fr.jetoile.hadoopunit.HadoopBootstrap;
 import fr.jetoile.hadoopunit.HadoopUnitConfig;
 import fr.jetoile.hadoopunit.exception.BootstrapException;
+import fr.jetoile.hadoopunit.exception.NotFoundServiceException;
 import info.archinnov.achilles.embedded.CassandraEmbeddedServerBuilder;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.thrift.transport.TTransportException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-
-//import org.cassandraunit.dataset.CQLDataSet;
-//import org.cassandraunit.utils.EmbeddedCassandraServerHelper;
+import java.util.Map;
 
 public class CassandraBootstrap implements Bootstrap {
     final public static String NAME = Component.CASSANDRA.name();
 
-    final private Logger LOGGER = LoggerFactory.getLogger(CassandraBootstrap.class);
+    static final private Logger LOGGER = LoggerFactory.getLogger(CassandraBootstrap.class);
 
     private State state = State.STOPPED;
 
@@ -80,6 +81,16 @@ private void loadConfig() throws BootstrapException {
         ip = configuration.getString(HadoopUnitConfig.CASSANDRA_IP_KEY);
     }
 
+    @Override
+    public void loadConfig(Map<String, String> configs) {
+        if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.CASSANDRA_PORT_KEY))) {
+            port = Integer.parseInt(configs.get(HadoopUnitConfig.CASSANDRA_PORT_KEY));
+        }
+        if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.CASSANDRA_IP_KEY))) {
+            ip = configs.get(HadoopUnitConfig.CASSANDRA_IP_KEY);
+        }
+    }
+
     private void build() throws InterruptedException, IOException, TTransportException {
         Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY)));
         Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/data"));

diff --git a/.../main/conf/hadoop-unit-default.properties → .../resources/hadoop-unit-default.properties b/.../main/conf/hadoop-unit-default.properties → .../resources/hadoop-unit-default.properties
diff --git a/...e-elasticsearch/src/main/conf/logback.xml → ...-cassandra/src/main/resources/logback.xml b/...e-elasticsearch/src/main/conf/logback.xml → ...-cassandra/src/main/resources/logback.xml
diff --git a/hadoop-unit-client/hadoop-unit-client-hdfs/pom.xml b/hadoop-unit-client/hadoop-unit-client-hdfs/pom.xml
@@ -4,57 +4,40 @@
     <parent>
         <artifactId>hadoop-unit-client</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>hadoop-unit-client-hdfs</artifactId>
     <name>hadoop-unit-client-hdfs</name>
 
     <dependencies>
-
         <dependency>
             <groupId>fr.jetoile.hadoop</groupId>
             <artifactId>hadoop-unit-commons</artifactId>
         </dependency>
 
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <artifactId>slf4j-log4j12</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
-        <!--<dependency>-->
-            <!--<groupId>org.apache.hadoop</groupId>-->
-            <!--<artifactId>hadoop-hdfs</artifactId>-->
-            <!--<version>2.7.1.2.3.2.0-2950</version>-->
-        <!--</dependency>-->
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
         </dependency>
 
-        <!-- Hive JDBC -->
-        <!--<dependency>-->
-            <!--<groupId>org.apache.hive</groupId>-->
-            <!--<artifactId>hive-jdbc</artifactId>-->
-        <!--</dependency>-->
-
         <dependency>
             <groupId>commons-configuration</groupId>
             <artifactId>commons-configuration</artifactId>
         </dependency>
 
-        <!--<dependency>-->
-            <!--<groupId>commons-collections</groupId>-->
-            <!--<artifactId>commons-collections</artifactId>-->
-        <!--</dependency>-->
-
-        <!--<dependency>-->
-            <!--<groupId>commons-io</groupId>-->
-            <!--<artifactId>commons-io</artifactId>-->
-        <!--</dependency>-->
-
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -64,13 +47,6 @@
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
         </dependency>
-
-        <!--<dependency>-->
-            <!--<groupId>com.ninja-squad</groupId>-->
-            <!--<artifactId>DbSetup</artifactId>-->
-        <!--</dependency>-->
-
-
     </dependencies>
 
 

diff --git a/hadoop-unit-client/hadoop-unit-client-hive/pom.xml b/hadoop-unit-client/hadoop-unit-client-hive/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>hadoop-unit-client</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -18,17 +18,6 @@
             <artifactId>hadoop-unit-commons</artifactId>
         </dependency>
 
-        <!--<dependency>-->
-            <!--<groupId>org.apache.hadoop</groupId>-->
-            <!--<artifactId>hadoop-hdfs</artifactId>-->
-            <!--<version>2.7.1.2.3.2.0-2950</version>-->
-        <!--</dependency>-->
-
-        <!--<dependency>-->
-            <!--<groupId>org.apache.hadoop</groupId>-->
-            <!--<artifactId>hadoop-common</artifactId>-->
-        <!--</dependency>-->
-
         <dependency>
             <groupId>commons-configuration</groupId>
             <artifactId>commons-configuration</artifactId>

diff --git a/hadoop-unit-client/hadoop-unit-client-kafka/pom.xml b/hadoop-unit-client/hadoop-unit-client-kafka/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hadoop-unit-client</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hadoop-unit-client/hadoop-unit-client-solrcloud/pom.xml b/hadoop-unit-client/hadoop-unit-client-solrcloud/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hadoop-unit-client</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hadoop-unit-client/hadoop-unit-client-spark/pom.xml b/hadoop-unit-client/hadoop-unit-client-spark/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hadoop-unit-client</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -14,7 +14,6 @@
 
     <dependencies>
 
-
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.10</artifactId>

diff --git a/hadoop-unit-client/pom.xml b/hadoop-unit-client/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>hadoop-unit</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hadoop-unit-commons/pom.xml b/hadoop-unit-commons/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hadoop-unit</artifactId>
         <groupId>fr.jetoile.hadoop</groupId>
-        <version>1.6-SNAPSHOT</version>
+        <version>2.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -19,26 +19,11 @@
             <artifactId>commons-configuration</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>commons-collections</groupId>
-            <artifactId>commons-collections</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
@@ -70,11 +55,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.eclipse.jetty</groupId>
-            <artifactId>jetty-util</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>

diff --git a/hadoop-unit-commons/src/main/java/com/github/sakserv/minicluster/util/WindowsLibsUtils.java b/hadoop-unit-commons/src/main/java/com/github/sakserv/minicluster/util/WindowsLibsUtils.java
@@ -12,43 +12,6 @@ public class WindowsLibsUtils {
     private static final Logger LOG = LoggerFactory.getLogger(WindowsLibsUtils.class);
 
     public static void setHadoopHome() {
-
-        // Set hadoop.home.dir to point to the windows lib dir
-        if (System.getProperty("os.name").startsWith("Windows")) {
-
-            String windowsLibDir = getHadoopHome();
-
-            LOG.info("WINDOWS: Setting hadoop.home.dir: {}", windowsLibDir);
-            System.setProperty("hadoop.home.dir", windowsLibDir);
-            System.load(new File(windowsLibDir + Path.SEPARATOR + "bin" + Path.SEPARATOR + "hadoop.dll").getAbsolutePath());
-            System.load(new File(windowsLibDir + Path.SEPARATOR + "bin" + Path.SEPARATOR + "hdfs.dll").getAbsolutePath());
-
-        }
-    }
-
-    public static String getHadoopHome() {
-
-        LOG.info("HADOOP_HOME: " + System.getenv("HADOOP_HOME"));
-
-        if(System.getenv("HADOOP_HOME") != null) {
-            return System.getenv("HADOOP_HOME");
-        } else {
-            if (System.getProperty("HADOOP_HOME") != null) {
-                return System.getProperty("HADOOP_HOME");
-            } else {
-                File windowsLibDir = new File("." + Path.SEPARATOR + "windows_libs" +
-                        Path.SEPARATOR + System.getProperty("hdp.release.version"));
-
-                if (!windowsLibDir.exists()) {
-                    windowsLibDir = new File(".." + Path.SEPARATOR + windowsLibDir);
-                    if (!windowsLibDir.exists()) {
-                        LOG.error("WINDOWS: ERROR: Could not find windows native libs");
-                    }
-                }
-                return windowsLibDir.getAbsolutePath();
-            }
-        }
-
     }
 
 }
diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java
@@ -18,29 +18,40 @@
  * Warning : this list should be sorted
  */
 public enum Component {
-    ZOOKEEPER("zookeeper"),
-    HDFS("hdfs"),
-    HIVEMETA("hivemeta"),
-    HIVESERVER2("hiveserver2"),
-    KAFKA("kafka"),
-    HBASE("hbase"),
-    OOZIE("oozie"),
-    SOLRCLOUD("solrcloud"),
-    SOLR("solr"),
-    CASSANDRA("cassandra"),
-    MONGODB("mongodb"),
-    ELASTICSEARCH("elastic"),
-    NEO4J("neo4j");
+    HDFS("hdfs", "fr.jetoile.hadoopunit.component.HdfsBootstrap", "hdfs.artifact"),
+    ZOOKEEPER("zookeeper", "fr.jetoile.hadoopunit.component.ZookeeperBootstrap", "zookeeper.artifact"),
+    HIVEMETA("hivemeta", "fr.jetoile.hadoopunit.component.HiveMetastoreBootstrap", "hivemeta.artifact"),
+    HIVESERVER2("hiveserver2", "fr.jetoile.hadoopunit.component.HiveServer2Bootstrap", "hiveserver2.artifact"),
+    KAFKA("kafka", "fr.jetoile.hadoopunit.component.KafkaBootstrap", "kafka.artifact"),
+    HBASE("hbase", "fr.jetoile.hadoopunit.component.HBaseBootstrap", "hbase.artifact"),
+    OOZIE("oozie", "fr.jetoile.hadoopunit.component.OozieBootstrap", "oozie.artifact"),
+    SOLRCLOUD("solrcloud", "fr.jetoile.hadoopunit.component.SolrCloudBootstrap", "solrcloud.artifact"),
+    SOLR("solr", "fr.jetoile.hadoopunit.component.SolrBootstrap", "solr.artifact"),
+    CASSANDRA("cassandra", "fr.jetoile.hadoopunit.component.CassandraBootstrap", "cassandra.artifact"),
+    MONGODB("mongodb", "fr.jetoile.hadoopunit.component.MongoDbBootstrap", "mongodb.artifact"),
+    ELASTICSEARCH("elasticsearch", "fr.jetoile.hadoopunit.component.ElasticSearchBootstrap", "elasticsearch.artifact"),
+    NEO4J("neo4j", "fr.jetoile.hadoopunit.component.Neo4jBootstrap", "neo4j.artifact");
 
     private String key;
+    private String mainClass;
+    private String artifactKey;
 
-    Component(String key) {
+    Component(String key, String mainClass, String artifactKey) {
         this.key = key;
+        this.mainClass = mainClass;
+        this.artifactKey = artifactKey;
     }
 
     public String getKey() {
         return key;
     }
 
+    public String getMainClass() {
+        return mainClass;
+    }
+
+    public String getArtifactKey() {
+        return artifactKey;
+    }
 }