urls = new ArrayList();
+
+ urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("log4j.xml"));
+ urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("logback.xml"));
+
+ if ("hiveserver2".equalsIgnoreCase(c)) {
+ urls.add(WriterAppender.class.getProtectionDomain().getCodeSource().getLocation());
+ }
+
+ if ("solrcloud".equalsIgnoreCase(c)) {
+ urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("solr"));
+ }
+
+ artifacts.forEach(f -> {
+ try {
+ urls.add(f.toURL());
+ } catch (MalformedURLException e) {
+ LOGGER.error("unable to find correct url for {}", f, e);
+ }
+ });
+
+ ClassLoader classloader = new URLClassLoader(
+ (URL[]) urls.toArray(new URL[0]),
+ ClassLoader.getSystemClassLoader().getParent());
+
+ // relative to that classloader, find the main class
+ Class mainClass = null;
+ try {
+ mainClass = classloader.loadClass(className);
+ } catch (ClassNotFoundException e) {
+ LOGGER.error("unable to load class", e);
+ }
+ Method main = null;
+
+
+ try {
+ Thread.currentThread().setContextClassLoader(classloader);
+
+ Object o = mainClass.getConstructor().newInstance();
+ main = mainClass.getMethod("start");
+ main.invoke(o);
+ return new ComponentProperties(o, mainClass);
+ } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
+ LOGGER.error("unable to reflect main", e);
+ }
+ return null;
+ }
+
+ private static class ComponentProperties {
+ private Object instance;
+ private Class mainClass;
+
+ public ComponentProperties(Object instance, Class mainClass) {
+ this.instance = instance;
+ this.mainClass = mainClass;
+ }
+
+ public Object getInstance() {
+ return instance;
+ }
+
+ public Class getMainClass() {
+ return mainClass;
+ }
+ }
+}
diff --git a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties
new file mode 100644
index 00000000..885b1e95
--- /dev/null
+++ b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties
@@ -0,0 +1,147 @@
+#HADOOP_HOME=/opt/hadoop
+
+zookeeper.artifact=fr.jetoile.hadoop:hadoop-unit-zookeeper:2.0-SNAPSHOT
+hdfs.artifact=fr.jetoile.hadoop:hadoop-unit-hdfs:2.0-SNAPSHOT
+hivemeta.artifact=fr.jetoile.hadoop:hadoop-unit-hive:2.0-SNAPSHOT
+hiveserver2.artifact=fr.jetoile.hadoop:hadoop-unit-hive:2.0-SNAPSHOT
+kafka.artifact=fr.jetoile.hadoop:hadoop-unit-kafka:2.0-SNAPSHOT
+hbase.artifact=fr.jetoile.hadoop:hadoop-unit-hbase:2.0-SNAPSHOT
+oozie.artifact=fr.jetoile.hadoop:hadoop-unit-oozie:2.0-SNAPSHOT
+solr.artifact=fr.jetoile.hadoop:hadoop-unit-solr:2.0-SNAPSHOT
+solrcloud.artifact=fr.jetoile.hadoop:hadoop-unit-solrcloud:2.0-SNAPSHOT
+cassandra.artifact=fr.jetoile.hadoop:hadoop-unit-cassandra:2.0-SNAPSHOT
+mongodb.artifact=fr.jetoile.hadoop:hadoop-unit-mongodb:2.0-SNAPSHOT
+elasticsearch.artifact=fr.jetoile.hadoop:hadoop-unit-elasticsearch:2.0-SNAPSHOT
+neo4j.artifact=fr.jetoile.hadoop:hadoop-unit-neo4j:2.0-SNAPSHOT
+
+maven.central.repo=https://repo.maven.apache.org/maven2/
+maven.local.repo=/home/khanh/.m2/repository
+
+# Zookeeper
+zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.host=127.0.0.1
+zookeeper.port=22010
+
+# Hive
+hive.scratch.dir=/tmp/hive_scratch_dir
+hive.warehouse.dir=/tmp/warehouse_dir
+
+# Hive Metastore
+hive.metastore.hostname=localhost
+hive.metastore.port=20102
+hive.metastore.derby.db.dir=metastore_db
+
+# Hive Server2
+hive.server2.hostname=localhost
+hive.server2.port=20103
+
+# Hive Test
+hive.test.database.name=default
+hive.test.table.name=test_table
+
+
+# HDFS
+hdfs.namenode.host=localhost
+hdfs.namenode.port=20112
+hdfs.namenode.http.port=50070
+hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.num.datanodes=1
+hdfs.enable.permissions=false
+hdfs.format=true
+hdfs.enable.running.user.as.proxy.user=true
+
+# HDFS Test
+hdfs.test.file=/tmp/testing
+hdfs.test.string=TESTING
+
+
+# HBase
+hbase.master.port=25111
+hbase.master.info.port=-1
+hbase.num.region.servers=1
+hbase.root.dir=/tmp/embedded_hbase
+hbase.znode.parent=/hbase-unsecure
+hbase.wal.replication.enabled=false
+
+# HBase Test
+hbase.test.table.name=hbase_test_table
+hbase.test.col.family.name=cf1
+hbase.test.col.qualifier.name=cq1
+hbase.test.num.rows.to.put=50
+
+# Kafka
+kafka.hostname=127.0.0.1
+kafka.port=20111
+
+# Kafka Test
+kafka.test.topic=testtopic
+kafka.test.message.count=10
+kafka.test.broker.id=1
+kafka.test.temp.dir=embedded_kafka
+
+#SolR + SolRCloud
+solr.dir=solr
+
+#SolR
+solr.collection.internal.name=collection1_shard1_replica1
+
+#SolRCloud
+solr.collection.name=collection1
+solr.cloud.port=8983
+
+
+
+
+
+# YARN
+yarn.num.node.managers=1
+yarn.num.local.dirs=1
+yarn.num.log.dirs=1
+yarn.resource.manager.address=localhost:37001
+yarn.resource.manager.hostname=localhost
+yarn.resource.manager.scheduler.address=localhost:37002
+yarn.resource.manager.resource.tracker.address=localhost:37003
+yarn.resource.manager.webapp.address=localhost:37004
+yarn.use.in.jvm.container.executor=false
+
+# MR
+mr.job.history.address=localhost:37005
+
+# Oozie
+oozie.tmp.dir=/tmp/oozie_tmp
+oozie.test.dir=/tmp/embedded_oozie
+oozie.home.dir=/tmp/oozie_home
+oozie.username=blah
+oozie.groupname=testgroup
+oozie.hdfs.share.lib.dir=/tmp/share_lib
+oozie.share.lib.create=true
+oozie.local.share.lib.cache.dir=/tmp/share_lib_cache
+oozie.purge.local.share.lib.cache=false
+oozie.sharelib.path=/home/khanh/github
+oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz
+oozie.port=20113
+oozie.host=localhost
+
+# ElasticSearch
+elasticsearch.ip=127.0.0.1
+elasticsearch.http.port=14433
+elasticsearch.tcp.port=14533
+elasticsearch.temp.dir=/tmp/elasticsearch
+elasticsearch.index.name=test_index
+elasticsearch.cluster.name=elasticsearch
+
+# MongoDB
+mongo.ip=127.0.0.1
+mongo.port=13333
+mongo.database.name=test_database
+mongo.collection.name=test_collection
+
+# Cassandra
+cassandra.ip=127.0.0.1
+cassandra.port=13433
+cassandra.temp.dir=/tmp/embedded_cassandra
+
+# Neo4j
+neo4j.ip=127.0.0.1
+neo4j.port=13533
+neo4j.temp.dir=/tmp/embedded_neo4j
\ No newline at end of file
diff --git a/hadoop-unit-standalone/src/main/resources/hadoop.properties b/hadoop-unit-standalone/src/main/resources/hadoop.properties
new file mode 100755
index 00000000..d595f3b7
--- /dev/null
+++ b/hadoop-unit-standalone/src/main/resources/hadoop.properties
@@ -0,0 +1,13 @@
+zookeeper=true
+hdfs=true
+hivemeta=true
+hiveserver2=true
+#kafka=true
+#hbase=true
+#solrcloud=true
+#oozie=true
+#mongodb=true
+#cassandra=true
+#elasticsearch=true
+#neo4j=true
+
diff --git a/hadoop-unit-standalone/src/main/resources/log4j.xml b/hadoop-unit-standalone/src/main/resources/log4j.xml
new file mode 100644
index 00000000..ee21aca7
--- /dev/null
+++ b/hadoop-unit-standalone/src/main/resources/log4j.xml
@@ -0,0 +1,31 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/hadoop-unit-standalone/src/main/resources/logback.xml b/hadoop-unit-standalone/src/main/resources/logback.xml
new file mode 100755
index 00000000..8329aeb7
--- /dev/null
+++ b/hadoop-unit-standalone/src/main/resources/logback.xml
@@ -0,0 +1,17 @@
+
+
+
+ %d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/currency.xml b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/currency.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/currency.xml
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/currency.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/lang/stopwords_en.txt b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/lang/stopwords_en.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/lang/stopwords_en.txt
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/lang/stopwords_en.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/protwords.txt b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/protwords.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/protwords.txt
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/protwords.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/schema.xml b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/schema.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/schema.xml
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/schema.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/solrconfig.xml b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/solrconfig.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/solrconfig.xml
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/solrconfig.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/stopwords.txt b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/stopwords.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/stopwords.txt
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/stopwords.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/synonyms.txt b/hadoop-unit-standalone/src/main/resources/solr/collection1/conf/synonyms.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/conf/synonyms.txt
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/conf/synonyms.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/core.properties b/hadoop-unit-standalone/src/main/resources/solr/collection1/core.properties
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/collection1/core.properties
rename to hadoop-unit-standalone/src/main/resources/solr/collection1/core.properties
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/solr.xml b/hadoop-unit-standalone/src/main/resources/solr/solr.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/main/conf/solr/solr.xml
rename to hadoop-unit-standalone/src/main/resources/solr/solr.xml
diff --git a/hadoop-unit-standalone/src/test/java/com/google/common/base/Stopwatch.java b/hadoop-unit-standalone/src/test/java/com/google/common/base/Stopwatch.java
new file mode 100644
index 00000000..7c6d8e0f
--- /dev/null
+++ b/hadoop-unit-standalone/src/test/java/com/google/common/base/Stopwatch.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright (C) 2008 The Guava Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.common.base;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+import com.google.common.annotations.Beta;
+import com.google.common.annotations.GwtCompatible;
+import com.google.common.annotations.GwtIncompatible;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An object that measures elapsed time in nanoseconds. It is useful to measure
+ * elapsed time using this class instead of direct calls to {@link
+ * System#nanoTime} for a few reasons:
+ *
+ *
+ * - An alternate time source can be substituted, for testing or performance
+ * reasons.
+ *
- As documented by {@code nanoTime}, the value returned has no absolute
+ * meaning, and can only be interpreted as relative to another timestamp
+ * returned by {@code nanoTime} at a different time. {@code Stopwatch} is a
+ * more effective abstraction because it exposes only these relative values,
+ * not the absolute ones.
+ *
+ *
+ * Basic usage:
+ *
+ * Stopwatch stopwatch = new Stopwatch().{@link #start start}();
+ * doSomething();
+ * stopwatch.{@link #stop stop}(); // optional
+ *
+ * long millis = stopwatch.{@link #elapsedMillis elapsedMillis}();
+ *
+ * log.info("that took: " + stopwatch); // formatted string like "12.3 ms"
+ *
+ *
+ * Stopwatch methods are not idempotent; it is an error to start or stop a
+ * stopwatch that is already in the desired state.
+ *
+ *
When testing code that uses this class, use the {@linkplain
+ * #Stopwatch(Ticker) alternate constructor} to supply a fake or mock ticker.
+ * This allows you to
+ * simulate any valid behavior of the stopwatch.
+ *
+ *
Note: This class is not thread-safe.
+ *
+ * @author Kevin Bourrillion
+ * @since 10.0
+ */
+
+/**
+ * for hbase test, guava 12 should be used because of conflict with cassandra driver
+ */
+@Beta
+@GwtCompatible(emulated=true)
+public final class Stopwatch {
+ private final Ticker ticker;
+ private boolean isRunning;
+ private long elapsedNanos;
+ private long startTick;
+
+ /**
+ * Creates (but does not start) a new stopwatch using {@link System#nanoTime}
+ * as its time source.
+ */
+ public Stopwatch() {
+ this(Ticker.systemTicker());
+ }
+
+ /**
+ * Creates (but does not start) a new stopwatch, using the specified time
+ * source.
+ */
+ public Stopwatch(Ticker ticker) {
+ this.ticker = checkNotNull(ticker);
+ }
+
+ /**
+ * Returns {@code true} if {@link #start()} has been called on this stopwatch,
+ * and {@link #stop()} has not been called since the last call to {@code
+ * start()}.
+ */
+ public boolean isRunning() {
+ return isRunning;
+ }
+
+ /**
+ * Starts the stopwatch.
+ *
+ * @return this {@code Stopwatch} instance
+ * @throws IllegalStateException if the stopwatch is already running.
+ */
+ public Stopwatch start() {
+ checkState(!isRunning);
+ isRunning = true;
+ startTick = ticker.read();
+ return this;
+ }
+
+ /**
+ * Stops the stopwatch. Future reads will return the fixed duration that had
+ * elapsed up to this point.
+ *
+ * @return this {@code Stopwatch} instance
+ * @throws IllegalStateException if the stopwatch is already stopped.
+ */
+ public Stopwatch stop() {
+ long tick = ticker.read();
+ checkState(isRunning);
+ isRunning = false;
+ elapsedNanos += tick - startTick;
+ return this;
+ }
+
+ /**
+ * Sets the elapsed time for this stopwatch to zero,
+ * and places it in a stopped state.
+ *
+ * @return this {@code Stopwatch} instance
+ */
+ public Stopwatch reset() {
+ elapsedNanos = 0;
+ isRunning = false;
+ return this;
+ }
+
+ private long elapsedNanos() {
+ return isRunning ? ticker.read() - startTick + elapsedNanos : elapsedNanos;
+ }
+
+ /**
+ * Returns the current elapsed time shown on this stopwatch, expressed
+ * in the desired time unit, with any fraction rounded down.
+ *
+ *
Note that the overhead of measurement can be more than a microsecond, so
+ * it is generally not useful to specify {@link TimeUnit#NANOSECONDS}
+ * precision here.
+ */
+ public long elapsedTime(TimeUnit desiredUnit) {
+ return desiredUnit.convert(elapsedNanos(), NANOSECONDS);
+ }
+
+ /**
+ * Returns the current elapsed time shown on this stopwatch, expressed
+ * in milliseconds, with any fraction rounded down. This is identical to
+ * {@code elapsedTime(TimeUnit.MILLISECONDS}.
+ */
+ public long elapsedMillis() {
+ return elapsedTime(MILLISECONDS);
+ }
+
+ /**
+ * Returns a string representation of the current elapsed time; equivalent to
+ * {@code toString(4)} (four significant figures).
+ */
+ @GwtIncompatible("String.format()")
+ @Override public String toString() {
+ return toString(4);
+ }
+
+ /**
+ * Returns a string representation of the current elapsed time, choosing an
+ * appropriate unit and using the specified number of significant figures.
+ * For example, at the instant when {@code elapsedTime(NANOSECONDS)} would
+ * return {1234567}, {@code toString(4)} returns {@code "1.235 ms"}.
+ */
+ @GwtIncompatible("String.format()")
+ public String toString(int significantDigits) {
+ long nanos = elapsedNanos();
+
+ TimeUnit unit = chooseUnit(nanos);
+ double value = (double) nanos / NANOSECONDS.convert(1, unit);
+
+ // Too bad this functionality is not exposed as a regular method call
+ return String.format("%." + significantDigits + "g %s",
+ value, abbreviate(unit));
+ }
+
+ private static TimeUnit chooseUnit(long nanos) {
+ if (SECONDS.convert(nanos, NANOSECONDS) > 0) {
+ return SECONDS;
+ }
+ if (MILLISECONDS.convert(nanos, NANOSECONDS) > 0) {
+ return MILLISECONDS;
+ }
+ if (MICROSECONDS.convert(nanos, NANOSECONDS) > 0) {
+ return MICROSECONDS;
+ }
+ return NANOSECONDS;
+ }
+
+ private static String abbreviate(TimeUnit unit) {
+ switch (unit) {
+ case NANOSECONDS:
+ return "ns";
+ case MICROSECONDS:
+ return "\u03bcs"; // μs
+ case MILLISECONDS:
+ return "ms";
+ case SECONDS:
+ return "s";
+ default:
+ throw new AssertionError();
+ }
+ }
+
+
+ /**
+ * Creates (but does not start) a new stopwatch using {@link System#nanoTime}
+ * as its time source.
+ *
+ * @since 15.0
+ */
+ public static Stopwatch createUnstarted() {
+ return new Stopwatch();
+ }
+
+ /**
+ * Creates (but does not start) a new stopwatch, using the specified time
+ * source.
+ *
+ * @since 15.0
+ */
+ public static Stopwatch createUnstarted(Ticker ticker) {
+ return new Stopwatch(ticker);
+ }
+
+ /**
+ * Creates (and starts) a new stopwatch using {@link System#nanoTime}
+ * as its time source.
+ *
+ * @since 15.0
+ */
+ public static Stopwatch createStarted() {
+ return new Stopwatch().start();
+ }
+
+ /**
+ * Creates (and starts) a new stopwatch, using the specified time
+ * source.
+ *
+ * @since 15.0
+ */
+ public static Stopwatch createStarted(Ticker ticker) {
+ return new Stopwatch(ticker).start();
+ }
+
+ /**
+ * Returns the current elapsed time shown on this stopwatch, expressed
+ * in the desired time unit, with any fraction rounded down.
+ *
+ *
Note that the overhead of measurement can be more than a microsecond, so
+ * it is generally not useful to specify {@link TimeUnit#NANOSECONDS}
+ * precision here.
+ *
+ * @since 14.0 (since 10.0 as {@code elapsedTime()})
+ */
+ public long elapsed(TimeUnit desiredUnit) {
+ return desiredUnit.convert(elapsedNanos(), NANOSECONDS);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-unit-commons/src/main/java/com/google/common/io/Closeables.java b/hadoop-unit-standalone/src/test/java/com/google/common/io/Closeables.java
similarity index 94%
rename from hadoop-unit-commons/src/main/java/com/google/common/io/Closeables.java
rename to hadoop-unit-standalone/src/test/java/com/google/common/io/Closeables.java
index 3bf3fcc4..ffb1c051 100644
--- a/hadoop-unit-commons/src/main/java/com/google/common/io/Closeables.java
+++ b/hadoop-unit-standalone/src/test/java/com/google/common/io/Closeables.java
@@ -1,7 +1,3 @@
-//
-// Source code recreated from a .class file by IntelliJ IDEA
-// (powered by Fernflower decompiler)
-//
package com.google.common.io;
@@ -68,4 +64,4 @@ public static void closeQuietly(@Nullable Closeable closeable) {
logger.log(Level.SEVERE, "IOException should not have been thrown.", e);
}
}
-}
+}
\ No newline at end of file
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java b/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java
similarity index 81%
rename from hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
rename to hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java
index ff903192..7dcd6262 100644
--- a/hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
+++ b/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/ManualIntegrationBootstrapTest.java
@@ -19,9 +19,6 @@
import com.datastax.driver.core.Session;
import com.mongodb.*;
import fr.jetoile.hadoopunit.HadoopUnitConfig;
-import fr.jetoile.hadoopunit.HadoopBootstrap;
-import fr.jetoile.hadoopunit.Utils;
-import fr.jetoile.hadoopunit.component.OozieBootstrap;
import fr.jetoile.hadoopunit.exception.BootstrapException;
import fr.jetoile.hadoopunit.exception.NotFoundServiceException;
import fr.jetoile.hadoopunit.test.hdfs.HdfsUtils;
@@ -43,6 +40,11 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowJob;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.zookeeper.KeeperException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -53,25 +55,30 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.junit.*;
+import org.neo4j.driver.v1.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.*;
-import java.sql.Connection;
import java.sql.*;
-import java.util.*;
-import java.util.Date;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
-import static junit.framework.TestCase.assertNotNull;
import static org.fest.assertions.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
-public class IntegrationBootstrapTest {
+@Ignore
+public class ManualIntegrationBootstrapTest {
static private Configuration configuration;
- static private Logger LOGGER = LoggerFactory.getLogger(IntegrationBootstrapTest.class);
+ static private Logger LOGGER = LoggerFactory.getLogger(ManualIntegrationBootstrapTest.class);
@BeforeClass
@@ -81,14 +88,39 @@ public static void setup() throws BootstrapException {
} catch (ConfigurationException e) {
throw new BootstrapException("bad config", e);
}
-
- HadoopBootstrap.INSTANCE.startAll();
}
@AfterClass
public static void tearDown() throws BootstrapException {
- HadoopBootstrap.INSTANCE.stopAll();
+ }
+
+ @Test
+ public void solrCloudShouldStart() throws IOException, SolrServerException, KeeperException, InterruptedException {
+
+ String collectionName = configuration.getString(HadoopUnitConfig.SOLR_COLLECTION_NAME);
+
+ String zkHostString = configuration.getString(HadoopUnitConfig.ZOOKEEPER_HOST_KEY) + ":" + configuration.getInt(HadoopUnitConfig.ZOOKEEPER_PORT_KEY);
+ CloudSolrClient client = new CloudSolrClient(zkHostString);
+
+ for (int i = 0; i < 1000; ++i) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("cat", "book");
+ doc.addField("id", "book-" + i);
+ doc.addField("name", "The Legend of the Hobbit part " + i);
+ client.add(collectionName, doc);
+ if (i % 100 == 0) client.commit(collectionName); // periodically flush
+ }
+ client.commit("collection1");
+
+ SolrDocument collection1 = client.getById(collectionName, "book-1");
+
+ assertNotNull(collection1);
+
+ assertThat(collection1.getFieldValue("name")).isEqualTo("The Legend of the Hobbit part 1");
+
+
+ client.close();
}
@Test
@@ -101,7 +133,6 @@ public void kafkaShouldStart() throws Exception {
}
-
// Consumer
KafkaConsumerUtils.INSTANCE.consumeMessagesWithNewApi(configuration.getString(HadoopUnitConfig.KAFKA_TEST_TOPIC_KEY), 10);
@@ -121,7 +152,6 @@ private String generateMessage(int i) {
}
@Test
- @Ignore
public void hiveServer2ShouldStart() throws InterruptedException, ClassNotFoundException, SQLException {
// assertThat(Utils.available("127.0.0.1", 20103)).isFalse();
@@ -198,12 +228,12 @@ public void hiveServer2ShouldStart() throws InterruptedException, ClassNotFoundE
@Test
public void hdfsShouldStart() throws Exception {
- assertThat(Utils.available("127.0.0.1", configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_HTTP_PORT_KEY))).isFalse();
-
+// assertThat(Utils.available("127.0.0.1", configuration.getInt(Config.HDFS_NAMENODE_HTTP_PORT_KEY))).isFalse();
+//
// org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
-// conf.set("fs.default.name", "hdfs://127.0.0.1:" + configuration.getInt(Config.HDFS_NAMENODE_PORT_KEY));
+// conf.set("fs.default.name", "hdfs://127.0.0.1:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY));
//
-// URI uri = URI.create ("hdfs://127.0.0.1:" + configuration.getInt(Config.HDFS_NAMENODE_PORT_KEY));
+// URI uri = URI.create ("hdfs://127.0.0.1:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY));
//
// FileSystem hdfsFsHandle = FileSystem.get (uri, conf);
FileSystem hdfsFsHandle = HdfsUtils.INSTANCE.getFileSystem();
@@ -220,18 +250,17 @@ public void hdfsShouldStart() throws Exception {
hdfsFsHandle.close();
URL url = new URL(
- String.format( "http://localhost:%s/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest",
- configuration.getInt( HadoopUnitConfig.HDFS_NAMENODE_HTTP_PORT_KEY ) ) );
+ String.format("http://localhost:%s/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest",
+ configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_HTTP_PORT_KEY)));
URLConnection connection = url.openConnection();
- connection.setRequestProperty( "Accept-Charset", "UTF-8" );
- BufferedReader response = new BufferedReader( new InputStreamReader( connection.getInputStream() ) );
+ connection.setRequestProperty("Accept-Charset", "UTF-8");
+ BufferedReader response = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String line = response.readLine();
response.close();
assertThat("{\"Path\":\"/user/guest\"}").isEqualTo(line);
}
-
@Test
public void hBaseShouldStart() throws Exception {
@@ -263,18 +292,20 @@ public void hBaseShouldStart() throws Exception {
}
+
@Test
public void oozieShouldStart() throws Exception {
+
LOGGER.info("OOZIE: Test Submit Workflow Start");
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
conf.set("fs.default.name", "hdfs://127.0.0.1:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY));
- URI uri = URI.create ("hdfs://127.0.0.1:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY));
+ URI uri = URI.create("hdfs://127.0.0.1:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY));
- FileSystem hdfsFs = FileSystem.get (uri, conf);
+ FileSystem hdfsFs = FileSystem.get(uri, conf);
- OozieClient oozieClient = new OozieClient("http://" + configuration.getString(OozieBootstrap.OOZIE_HOST) + ":" + configuration.getInt(OozieBootstrap.OOZIE_PORT) + "/oozie");
+ OozieClient oozieClient = new OozieClient("http://" + configuration.getString(HadoopUnitConfig.OOZIE_HOST) + ":" + configuration.getInt(HadoopUnitConfig.OOZIE_PORT) + "/oozie");
Path appPath = new Path(hdfsFs.getHomeDirectory(), "testApp");
hdfsFs.mkdirs(new Path(appPath, "lib"));
@@ -298,7 +329,7 @@ public void oozieShouldStart() throws Exception {
//submit and check
final String jobId = oozieClient.submit(oozieConf);
WorkflowJob wf = oozieClient.getJobInfo(jobId);
- Assert.assertNotNull(wf);
+ assertNotNull(wf);
assertEquals(WorkflowJob.Status.PREP, wf.getStatus());
LOGGER.info("OOZIE: Workflow: {}", wf.toString());
@@ -306,6 +337,39 @@ public void oozieShouldStart() throws Exception {
}
+ private static void createHbaseTable(String tableName, String colFamily,
+ org.apache.hadoop.conf.Configuration configuration) throws Exception {
+
+ final HBaseAdmin admin = new HBaseAdmin(configuration);
+ HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
+ HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(colFamily);
+
+ hTableDescriptor.addFamily(hColumnDescriptor);
+ admin.createTable(hTableDescriptor);
+ }
+
+ private static void putRow(String tableName, String colFamName, String rowKey, String colQualifier, String value,
+ org.apache.hadoop.conf.Configuration configuration) throws Exception {
+ HTable table = new HTable(configuration, tableName);
+ Put put = new Put(Bytes.toBytes(rowKey));
+ put.add(Bytes.toBytes(colFamName), Bytes.toBytes(colQualifier), Bytes.toBytes(value));
+ table.put(put);
+ table.flushCommits();
+ table.close();
+ }
+
+ private static Result getRow(String tableName, String colFamName, String rowKey, String colQualifier,
+ org.apache.hadoop.conf.Configuration configuration) throws Exception {
+ Result result;
+ HTable table = new HTable(configuration, tableName);
+ Get get = new Get(Bytes.toBytes(rowKey));
+ get.addColumn(Bytes.toBytes(colFamName), Bytes.toBytes(colQualifier));
+ get.setMaxVersions(1);
+ result = table.get(get);
+ return result;
+ }
+
+
@Test
public void mongodbShouldStart() throws UnknownHostException {
MongoClient mongo = new MongoClient(configuration.getString(HadoopUnitConfig.MONGO_IP_KEY), configuration.getInt(HadoopUnitConfig.MONGO_PORT_KEY));
@@ -314,12 +378,12 @@ public void mongodbShouldStart() throws UnknownHostException {
DBCollection col = db.createCollection(configuration.getString(HadoopUnitConfig.MONGO_COLLECTION_NAME_KEY),
new BasicDBObject());
- col.save(new BasicDBObject("testDoc", new Date()));
+ col.save(new BasicDBObject("testDoc", new java.util.Date()));
LOGGER.info("MONGODB: Number of items in collection: {}", col.count());
assertEquals(1, col.count());
DBCursor cursor = col.find();
- while(cursor.hasNext()) {
+ while (cursor.hasNext()) {
LOGGER.info("MONGODB: Document output: {}", cursor.next());
}
cursor.close();
@@ -365,62 +429,42 @@ public void elasticSearchShouldStartWithRealDriver() throws NotFoundServiceExcep
IndexResponse ir = client.prepareIndex("test_index", "type").setSource(jsonString).setId("2").execute().actionGet();
client.admin().indices().prepareRefresh("test_index").execute().actionGet();
- Assert.assertNotNull(ir);
+ assertNotNull(ir);
GetResponse gr = client.prepareGet("test_index", "type", "2").execute().actionGet();
- Assert.assertNotNull(gr);
+ assertNotNull(gr);
assertEquals(gr.getSourceAsString(), "{\"value\":\"value2\",\"size\":0.33,\"price\":3.0}");
}
- private final Map headers = new HashMap();
-
- protected HttpURLConnection createConnection(URL url, String method) throws IOException {
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setRequestMethod(method);
- if (method.equals("POST") || method.equals("PUT")) {
- conn.setDoOutput(true);
- }
- for (Map.Entry header : headers.entrySet()) {
- conn.setRequestProperty(header.getKey(), header.getValue());
+ @Test
+ public void neo4jShouldStartWithRealDriver() {
+
+ org.neo4j.driver.v1.Driver driver = GraphDatabase.driver(
+ "bolt://localhost:13533",
+ Config.build()
+ .withEncryptionLevel(Config.EncryptionLevel.NONE)
+ .toConfig()
+ );
+
+ List results = new ArrayList<>();
+ try (org.neo4j.driver.v1.Session session = driver.session()) {
+ session.run("CREATE (person:Person {name: {name}, title:'King'})", Values.parameters("name", "Arthur"));
+
+ StatementResult result = session.run("MATCH (a:Person) WHERE a.name = 'Arthur' RETURN a.name AS name, a.title AS title");
+ while (result.hasNext()) {
+ Record record = result.next();
+ results.add(record);
+ LOGGER.debug(record.get("title").asString() + " " + record.get("name").asString());
+ }
}
- return conn;
- }
-
- private static void createHbaseTable(String tableName, String colFamily,
- org.apache.hadoop.conf.Configuration configuration) throws Exception {
- final HBaseAdmin admin = new HBaseAdmin(configuration);
- HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
- HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(colFamily);
-
- hTableDescriptor.addFamily(hColumnDescriptor);
- admin.createTable(hTableDescriptor);
+ assertEquals(1, results.size());
+ assertEquals("King", results.get(0).get("title").asString());
+ assertEquals("Arthur", results.get(0).get("name").asString());
}
- private static void putRow(String tableName, String colFamName, String rowKey, String colQualifier, String value,
- org.apache.hadoop.conf.Configuration configuration) throws Exception {
- HTable table = new HTable(configuration, tableName);
- Put put = new Put(Bytes.toBytes(rowKey));
- put.add(Bytes.toBytes(colFamName), Bytes.toBytes(colQualifier), Bytes.toBytes(value));
- table.put(put);
- table.flushCommits();
- table.close();
- }
-
- private static Result getRow(String tableName, String colFamName, String rowKey, String colQualifier,
- org.apache.hadoop.conf.Configuration configuration) throws Exception {
- Result result;
- HTable table = new HTable(configuration, tableName);
- Get get = new Get(Bytes.toBytes(rowKey));
- get.addColumn(Bytes.toBytes(colFamName), Bytes.toBytes(colQualifier));
- get.setMaxVersions(1);
- result = table.get(get);
- return result;
- }
-
-
}
diff --git a/hadoop-unit-zookeeper/pom.xml b/hadoop-unit-zookeeper/pom.xml
index ca676d96..9f16588c 100644
--- a/hadoop-unit-zookeeper/pom.xml
+++ b/hadoop-unit-zookeeper/pom.xml
@@ -5,7 +5,7 @@
hadoop-unit
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
@@ -24,4 +24,5 @@
hadoop-unit-commons
+
\ No newline at end of file
diff --git a/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java b/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java
index 3032e631..10a42e00 100644
--- a/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java
+++ b/hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java
@@ -15,12 +15,15 @@
import com.github.sakserv.minicluster.impl.ZookeeperLocalCluster;
import fr.jetoile.hadoopunit.Component;
+import fr.jetoile.hadoopunit.HadoopBootstrap;
import fr.jetoile.hadoopunit.HadoopUnitConfig;
import fr.jetoile.hadoopunit.HadoopUtils;
import fr.jetoile.hadoopunit.exception.BootstrapException;
+import fr.jetoile.hadoopunit.exception.NotFoundServiceException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,11 +31,12 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Map;
public class ZookeeperBootstrap implements Bootstrap {
final public static String NAME = Component.ZOOKEEPER.name();
- final private Logger LOGGER = LoggerFactory.getLogger(ZookeeperBootstrap.class);
+ final private static Logger LOGGER = LoggerFactory.getLogger(ZookeeperBootstrap.class);
private ZookeeperLocalCluster zookeeperLocalCluster;
@@ -69,7 +73,7 @@ public String getProperties() {
}
private void loadConfig() throws BootstrapException {
- HadoopUtils.setHadoopHome();
+ HadoopUtils.INSTANCE.setHadoopHome();
try {
configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
} catch (ConfigurationException e) {
@@ -81,6 +85,18 @@ private void loadConfig() throws BootstrapException {
}
+ @Override
+ public void loadConfig(Map configs) {
+ if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ZOOKEEPER_PORT_KEY))) {
+ port = Integer.parseInt(configs.get(HadoopUnitConfig.ZOOKEEPER_PORT_KEY));
+ }
+ if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ZOOKEEPER_TEMP_DIR_KEY))) {
+ localDir = configs.get(HadoopUnitConfig.ZOOKEEPER_TEMP_DIR_KEY);
+ }
+ if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ZOOKEEPER_HOST_KEY))) {
+ host = configs.get(HadoopUnitConfig.ZOOKEEPER_HOST_KEY);
+ }
+ }
private void init() {
Path path = Paths.get(localDir);
@@ -138,6 +154,4 @@ public Bootstrap stop() {
public org.apache.hadoop.conf.Configuration getConfiguration() {
throw new UnsupportedOperationException("the method getConfiguration can not be called on ZookeeperBootstrap");
}
-
-
}
diff --git a/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties
new file mode 100644
index 00000000..ad8559d8
--- /dev/null
+++ b/hadoop-unit-zookeeper/src/main/resources/hadoop-unit-default.properties
@@ -0,0 +1,106 @@
+#HADOOP_HOME=/opt/hadoop
+
+# Zookeeper
+zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.host=127.0.0.1
+zookeeper.port=22010
+
+# Hive
+hive.scratch.dir=/tmp/hive_scratch_dir
+hive.warehouse.dir=/tmp/warehouse_dir
+
+# Hive Metastore
+hive.metastore.hostname=localhost
+hive.metastore.port=20102
+hive.metastore.derby.db.dir=metastore_db
+
+# Hive Server2
+hive.server2.hostname=localhost
+hive.server2.port=20103
+
+# Hive Test
+hive.test.database.name=default
+hive.test.table.name=test_table
+
+
+# HDFS
+hdfs.namenode.port=20112
+hdfs.namenode.http.port=50070
+hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.num.datanodes=1
+hdfs.enable.permissions=false
+hdfs.format=true
+hdfs.enable.running.user.as.proxy.user=true
+
+# HDFS Test
+hdfs.test.file=/tmp/testing
+hdfs.test.string=TESTING
+
+
+# HBase
+hbase.master.port=25111
+hbase.master.info.port=-1
+hbase.num.region.servers=1
+hbase.root.dir=/tmp/embedded_hbase
+hbase.znode.parent=/hbase-unsecure
+hbase.wal.replication.enabled=false
+
+# HBase Test
+hbase.test.table.name=hbase_test_table
+hbase.test.col.family.name=cf1
+hbase.test.col.qualifier.name=cq1
+hbase.test.num.rows.to.put=50
+
+# Kafka
+kafka.hostname=127.0.0.1
+kafka.port=20111
+
+# Kafka Test
+kafka.test.topic=testtopic
+kafka.test.message.count=10
+kafka.test.broker.id=1
+kafka.test.temp.dir=embedded_kafka
+
+#SolR + SolRCloud
+solr.dir=solr
+
+#SolR
+solr.collection.internal.name=collection1_shard1_replica1
+
+#SolRCloud
+solr.collection.name=collection1
+solr.cloud.port=8983
+
+
+# YARN
+yarn.num.node.managers=1
+yarn.num.local.dirs=1
+yarn.num.log.dirs=1
+yarn.resource.manager.address=localhost:37001
+yarn.resource.manager.hostname=localhost
+yarn.resource.manager.scheduler.address=localhost:37002
+yarn.resource.manager.resource.tracker.address=localhost:37003
+yarn.resource.manager.webapp.address=localhost:37004
+yarn.use.in.jvm.container.executor=false
+
+# MR
+mr.job.history.address=localhost:37005
+
+# Oozie
+oozie.test.dir=/tmp/embedded_oozie
+oozie.home.dir=/tmp/oozie_home
+oozie.username=blah
+oozie.groupname=testgroup
+oozie.hdfs.share.lib.dir=/tmp/share_lib
+oozie.share.lib.create=true
+oozie.local.share.lib.cache.dir=/tmp/share_lib_cache
+oozie.purge.local.share.lib.cache=false
+oozie.sharelib.path=/home/khanh/github
+oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz
+oozie.port=20113
+oozie.host=localhost
+
+# Neo4j
+neo4j.ip=127.0.0.1
+neo4j.port=13533
+neo4j.temp.dir=/tmp/embedded_neo4j
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 7d4201fc..a273d586 100755
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
fr.jetoile.hadoop
hadoop-unit
pom
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
hadoop-unit
https://github.com/jetoile/hadoop-unit
@@ -115,7 +115,7 @@
1.8
1.8
- 1.6-SNAPSHOT
+ ${project.version}
1.7.12
1.1.3
@@ -216,6 +216,10 @@
hadoop-unit-commons
${hadoop-unit.version}
+
+ com.microsoft.windowsazure.storage
+ microsoft-windowsazure-storage-sdk
+
servlet-api
javax.servlet
@@ -359,32 +363,6 @@
org.apache.hadoop
hadoop-common
${hadoop.version}
-
-
- httpclient
- org.apache.httpcomponents
-
-
- servlet-api
- javax.servlet
-
-
- org.eclipse.jetty
- jetty-util
-
-
- guava
- com.google.guava
-
-
- log4j
- log4j
-
-
- slf4j-log4j12
- org.slf4j
-
-
@@ -410,14 +388,6 @@
servlet-api
javax.servlet
-
- log4j
- log4j
-
-
- slf4j-log4j12
- org.slf4j
-
@@ -446,22 +416,6 @@
org.apache.hadoop
hadoop-annotations
-
- org.ow2.asm
- asm
-
-
- org.ow2.asm
- asm-commons
-
-
- jetty-util
- org.eclipse.jetty
-
-
- com.googlecode.concurrentlinkedhashmap
- concurrentlinkedhashmap-lru
-
log4j
log4j
@@ -485,44 +439,6 @@
com.github.sakserv
hadoop-mini-clusters-oozie
${hadoop-mini-clusters.version}
-
-
- servlet-api
- javax.servlet
-
-
- metrics-core
- com.codahale.metrics
-
-
- metrics-json
- com.codahale.metrics
-
-
- metrics-jvm
- com.codahale.metrics
-
-
- hive-jdbc
- org.apache.hive
-
-
- log4j
- log4j
-
-
- slf4j-simple
- org.slf4j
-
-
- slf4j-log4j12
- org.slf4j
-
-
- property-parser
- com.github.sakserv
-
-
@@ -606,14 +522,6 @@
hadoop-mini-clusters-hivemetastore
${hadoop-mini-clusters.version}
-
- log4j
- log4j
-
-
- slf4j-log4j12
- org.slf4j
-
@@ -622,10 +530,6 @@
hadoop-mini-clusters-hiveserver2
${hadoop-mini-clusters.version}
-
- jetty-all
- org.eclipse.jetty.aggregate
-
slf4j-log4j12
org.slf4j
@@ -637,36 +541,12 @@
com.github.sakserv
hadoop-mini-clusters-hdfs
${hadoop-mini-clusters.version}
-
-
- servlet-api
- javax.servlet
-
-
- log4j
- log4j
-
-
- slf4j-log4j12
- org.slf4j
-
-
- netty
- io.netty
-
-
com.github.sakserv
hadoop-mini-clusters-hbase
${hadoop-mini-clusters.version}
-
-
- servlet-api-2.5
- org.mortbay.jetty
-
-
@@ -674,14 +554,6 @@
spark-solr
${spark-solr.version}
-
- org.ow2.asm
- asm
-
-
- org.ow2.asm
- asm-commons
-
spark-mllib_2.10
org.apache.spark
@@ -828,7 +700,7 @@
junit
junit
- 4.11
+ 4.12
test
diff --git a/sample/all/pom.xml b/sample/all/pom.xml
new file mode 100644
index 00000000..25937859
--- /dev/null
+++ b/sample/all/pom.xml
@@ -0,0 +1,133 @@
+
+
+
+ sample
+ fr.jetoile.hadoop
+ 2.0-SNAPSHOT
+
+ 4.0.0
+
+ all
+
+
+ 2.11.3
+
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-commons
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
+ org.mortbay.jetty
+ org.eclipse.jetty
+
+
+ org.eclipse.jetty
+ jetty-util
+
+
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-hbase
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-hdfs
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-hive
+
+
+ jetty-all
+ org.eclipse.jetty.aggregate
+
+
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-kafka
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-oozie
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-zookeeper
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-mongodb
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-cassandra
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-solrcloud
+
+
+ jetty-util
+ org.eclipse.jetty
+
+
+
+
+
+ org.eclipse.jetty
+ jetty-util
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-client-kafka
+ test
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-client-solrcloud
+ test
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-client-hdfs
+ test
+
+
+
+ fr.jetoile.hadoop
+ hadoop-unit-client-hive
+ test
+
+
+
+ org.mongodb
+ mongo-java-driver
+ ${mongo-java-driver.version}
+ test
+
+
+
+
+
\ No newline at end of file
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java b/sample/all/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
similarity index 98%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
rename to sample/all/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
index d86b3533..6f5325ad 100644
--- a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
+++ b/sample/all/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java
@@ -18,11 +18,9 @@
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.Session;
import com.mongodb.*;
-import fr.jetoile.hadoopunit.HadoopUnitConfig;
import fr.jetoile.hadoopunit.HadoopBootstrap;
+import fr.jetoile.hadoopunit.HadoopUnitConfig;
import fr.jetoile.hadoopunit.Utils;
-import fr.jetoile.hadoopunit.component.OozieBootstrap;
-import fr.jetoile.hadoopunit.component.SolrCloudBootstrap;
import fr.jetoile.hadoopunit.exception.BootstrapException;
import fr.jetoile.hadoopunit.exception.NotFoundServiceException;
import fr.jetoile.hadoopunit.test.hdfs.HdfsUtils;
@@ -59,13 +57,14 @@
import java.net.*;
import java.sql.Connection;
import java.sql.*;
-import java.util.*;
import java.util.Date;
+import java.util.*;
import static junit.framework.TestCase.assertNotNull;
import static org.fest.assertions.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
+@Ignore
public class IntegrationBootstrapTest {
static private Configuration configuration;
@@ -93,7 +92,7 @@ public static void tearDown() throws BootstrapException {
@Test
public void solrCloudShouldStart() throws IOException, SolrServerException, KeeperException, InterruptedException {
- String collectionName = configuration.getString(SolrCloudBootstrap.SOLR_COLLECTION_NAME);
+ String collectionName = configuration.getString(HadoopUnitConfig.SOLR_COLLECTION_NAME);
String zkHostString = configuration.getString(HadoopUnitConfig.ZOOKEEPER_HOST_KEY) + ":" + configuration.getInt(HadoopUnitConfig.ZOOKEEPER_PORT_KEY);
CloudSolrClient client = new CloudSolrClient(zkHostString);
@@ -301,7 +300,7 @@ public void oozieShouldStart() throws Exception {
FileSystem hdfsFs = FileSystem.get (uri, conf);
- OozieClient oozieClient = new OozieClient("http://" + configuration.getString(OozieBootstrap.OOZIE_HOST) + ":" + configuration.getInt(OozieBootstrap.OOZIE_PORT) + "/oozie");
+ OozieClient oozieClient = new OozieClient("http://" + configuration.getString(HadoopUnitConfig.OOZIE_HOST) + ":" + configuration.getInt(HadoopUnitConfig.OOZIE_PORT) + "/oozie");
Path appPath = new Path(hdfsFs.getHomeDirectory(), "testApp");
hdfsFs.mkdirs(new Path(appPath, "lib"));
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/hadoop-unit-default.properties b/sample/all/src/test/resources/hadoop-unit-default.properties
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/hadoop-unit-default.properties
rename to sample/all/src/test/resources/hadoop-unit-default.properties
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/resources/hive-site.xml b/sample/all/src/test/resources/hive-site.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/resources/hive-site.xml
rename to sample/all/src/test/resources/hive-site.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/currency.xml b/sample/all/src/test/resources/solr/collection1/conf/currency.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/currency.xml
rename to sample/all/src/test/resources/solr/collection1/conf/currency.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt b/sample/all/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt
rename to sample/all/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/protwords.txt b/sample/all/src/test/resources/solr/collection1/conf/protwords.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/protwords.txt
rename to sample/all/src/test/resources/solr/collection1/conf/protwords.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/schema.xml b/sample/all/src/test/resources/solr/collection1/conf/schema.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/schema.xml
rename to sample/all/src/test/resources/solr/collection1/conf/schema.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/solrconfig.xml b/sample/all/src/test/resources/solr/collection1/conf/solrconfig.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/solrconfig.xml
rename to sample/all/src/test/resources/solr/collection1/conf/solrconfig.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/stopwords.txt b/sample/all/src/test/resources/solr/collection1/conf/stopwords.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/stopwords.txt
rename to sample/all/src/test/resources/solr/collection1/conf/stopwords.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/synonyms.txt b/sample/all/src/test/resources/solr/collection1/conf/synonyms.txt
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/conf/synonyms.txt
rename to sample/all/src/test/resources/solr/collection1/conf/synonyms.txt
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/core.properties b/sample/all/src/test/resources/solr/collection1/core.properties
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/collection1/core.properties
rename to sample/all/src/test/resources/solr/collection1/core.properties
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/solr.xml b/sample/all/src/test/resources/solr/solr.xml
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-solr/src/test/resources/solr/solr.xml
rename to sample/all/src/test/resources/solr/solr.xml
diff --git a/hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/resources/test.csv b/sample/all/src/test/resources/test.csv
similarity index 100%
rename from hadoop-unit-standalone/hadoop-unit-standalone-elasticsearch/src/test/resources/test.csv
rename to sample/all/src/test/resources/test.csv
diff --git a/sample/kafka-spark-streaming/pom.xml b/sample/kafka-spark-streaming/pom.xml
index 3f20c3e2..c467f505 100644
--- a/sample/kafka-spark-streaming/pom.xml
+++ b/sample/kafka-spark-streaming/pom.xml
@@ -5,7 +5,7 @@
sample
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
@@ -70,4 +70,66 @@
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ **/*IntegrationTest.java
+
+
+
+
+ integration-test
+
+ test
+
+ integration-test
+
+
+ none
+
+
+ **/*IntegrationTest.java
+
+
+
+
+
+
+
+ hadoop-unit-maven-plugin
+ fr.jetoile.hadoop
+ ${hadoop-unit.version}
+
+
+ start
+
+ embedded-start
+
+ pre-integration-test
+
+
+
+ /home/khanh/.m2/repository
+ https://repo.maven.apache.org/maven2/
+
+
+ ZOOKEEPER
+ fr.jetoile.hadoop:hadoop-unit-zookeeper:${hadoop-unit.version}
+
+
+ KAFKA
+ fr.jetoile.hadoop:hadoop-unit-kafka:${hadoop-unit.version}
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/sample/kafka-spark-streaming/src/test/java/fr/jetoile/hadoopunit/sample/kafka/SparkKafkaIntegrationTest.java b/sample/kafka-spark-streaming/src/test/java/fr/jetoile/hadoopunit/sample/kafka/SparkKafkaIntegrationTest.java
index e3791bea..30c11455 100644
--- a/sample/kafka-spark-streaming/src/test/java/fr/jetoile/hadoopunit/sample/kafka/SparkKafkaIntegrationTest.java
+++ b/sample/kafka-spark-streaming/src/test/java/fr/jetoile/hadoopunit/sample/kafka/SparkKafkaIntegrationTest.java
@@ -40,7 +40,6 @@
import java.util.*;
import java.util.regex.Pattern;
-@Ignore
public class SparkKafkaIntegrationTest implements Serializable {
private static Configuration configuration;
@@ -50,6 +49,7 @@ public static void setUp() throws ConfigurationException {
configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
}
+ @Ignore
@Test
public void stark_should_read_kafka() {
diff --git a/sample/kafka-stream/pom.xml b/sample/kafka-stream/pom.xml
index 178d8476..d18e7509 100644
--- a/sample/kafka-stream/pom.xml
+++ b/sample/kafka-stream/pom.xml
@@ -5,7 +5,7 @@
sample
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
@@ -49,4 +49,64 @@
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ **/*IntegrationTest.java
+
+
+
+
+ integration-test
+
+ test
+
+ integration-test
+
+
+ none
+
+
+ **/*IntegrationTest.java
+
+
+
+
+
+
+
+ hadoop-unit-maven-plugin
+ fr.jetoile.hadoop
+ ${hadoop-unit.version}
+
+
+ start
+
+ embedded-start
+
+ pre-integration-test
+
+
+
+ /home/khanh/.m2/repository
+ https://repo.maven.apache.org/maven2/
+
+
+ ZOOKEEPER
+ fr.jetoile.hadoop:hadoop-unit-zookeeper:${hadoop-unit.version}
+
+
+ KAFKA
+ fr.jetoile.hadoop:hadoop-unit-kafka:${hadoop-unit.version}
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobTest.java b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobIntegrationTest.java
similarity index 97%
rename from sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobTest.java
rename to sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobIntegrationTest.java
index 3eb63815..ed4f69b1 100644
--- a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobTest.java
+++ b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobIntegrationTest.java
@@ -27,8 +27,7 @@
import static org.junit.Assert.*;
-@Ignore
-public class KafkaStreamsJobTest {
+public class KafkaStreamsJobIntegrationTest {
private static Configuration configuration;
diff --git a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobWithUnitTest.java b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobWithUnitTest.java
index 874c5ffe..341ece7d 100644
--- a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobWithUnitTest.java
+++ b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsJobWithUnitTest.java
@@ -24,8 +24,10 @@
import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
+@Ignore
public class KafkaStreamsJobWithUnitTest {
private static Configuration configuration;
diff --git a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobTest.java b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobIntegrationTest.java
similarity index 97%
rename from sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobTest.java
rename to sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobIntegrationTest.java
index 5c7ad7f0..07985960 100644
--- a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobTest.java
+++ b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobIntegrationTest.java
@@ -28,7 +28,7 @@
import org.junit.Test;
@Ignore
-public class KafkaStreamsPipelineJobTest {
+public class KafkaStreamsPipelineJobIntegrationTest {
private static Configuration configuration;
diff --git a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobWithUnitTest.java b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobWithUnitTest.java
index 2bbe79f9..13a95589 100644
--- a/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobWithUnitTest.java
+++ b/sample/kafka-stream/src/test/java/fr/jetoile/hadoopunit/sample/kafka/KafkaStreamsPipelineJobWithUnitTest.java
@@ -25,6 +25,7 @@
import org.codehaus.jettison.json.JSONObject;
import org.junit.*;
+@Ignore
public class KafkaStreamsPipelineJobWithUnitTest {
private static Configuration configuration;
diff --git a/sample/parquet-spark/pom.xml b/sample/parquet-spark/pom.xml
index 6a439fd5..9dfcfbb4 100644
--- a/sample/parquet-spark/pom.xml
+++ b/sample/parquet-spark/pom.xml
@@ -5,7 +5,7 @@
sample
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
@@ -47,12 +47,22 @@
jackson-databind
com.fasterxml.jackson.core
+
+ slf4j-log4j12
+ org.slf4j
+
org.apache.spark
spark-hive_2.10
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
@@ -64,18 +74,105 @@
fr.jetoile.hadoop
hadoop-unit-commons
test
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
fr.jetoile.hadoop
hadoop-unit-client-hdfs
test
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
fr.jetoile.hadoop
hadoop-unit-client-hive
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ **/*IntegrationTest.java
+
+
+
+
+ integration-test
+
+ test
+
+ integration-test
+
+
+ none
+
+
+ **/*IntegrationTest.java
+
+
+
+
+
+
+
+ hadoop-unit-maven-plugin
+ fr.jetoile.hadoop
+ ${hadoop-unit.version}
+
+
+ start
+
+ embedded-start
+
+ pre-integration-test
+
+
+
+ /home/khanh/.m2/repository
+ https://repo.maven.apache.org/maven2/
+
+
+ HDFS
+ fr.jetoile.hadoop:hadoop-unit-hdfs:${hadoop-unit.version}
+
+
+ ZOOKEEPER
+ fr.jetoile.hadoop:hadoop-unit-zookeeper:${hadoop-unit.version}
+
+
+ HIVEMETA
+ fr.jetoile.hadoop:hadoop-unit-hive:${hadoop-unit.version}
+
+
+ HIVESERVER2
+ fr.jetoile.hadoop:hadoop-unit-hive:${hadoop-unit.version}
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/sample/parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/SparkJobTest.java b/sample/parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/SparkJobIntegrationTest.java
similarity index 96%
rename from sample/parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/SparkJobTest.java
rename to sample/parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/SparkJobIntegrationTest.java
index 3d1f8baa..3adc54ab 100644
--- a/sample/parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/SparkJobTest.java
+++ b/sample/parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/SparkJobIntegrationTest.java
@@ -45,10 +45,9 @@
import static com.ninja_squad.dbsetup.Operations.sql;
import static org.fest.assertions.Assertions.assertThat;
-@Ignore
-public class SparkJobTest {
+public class SparkJobIntegrationTest {
- static private Logger LOGGER = LoggerFactory.getLogger(SparkJobTest.class);
+ static private Logger LOGGER = LoggerFactory.getLogger(SparkJobIntegrationTest.class);
private static Configuration configuration;
@@ -86,7 +85,7 @@ public void before() throws IOException, URISyntaxException {
fileSystem.mkdirs(new Path("/khanh/test"));
fileSystem.mkdirs(new Path("/khanh/test_parquet"));
- fileSystem.copyFromLocalFile(new Path(SparkJobTest.class.getClassLoader().getResource("test.csv").toURI()), new Path("/khanh/test/test.csv"));
+ fileSystem.copyFromLocalFile(new Path(SparkJobIntegrationTest.class.getClassLoader().getResource("test.csv").toURI()), new Path("/khanh/test/test.csv"));
new HiveSetup(HiveConnectionUtils.INSTANCE.getDestination(), Operations.sequenceOf(CREATE_TABLES)).launch();
}
@@ -101,6 +100,7 @@ public void clean() throws IOException {
@Test
+ @Ignore
public void upload_file_into_hdfs_and_map_hive_should_success() throws SQLException {
Statement stmt = HiveConnectionUtils.INSTANCE.getConnection().createStatement();
diff --git a/sample/pom.xml b/sample/pom.xml
index e64cb9c0..d7ac175b 100644
--- a/sample/pom.xml
+++ b/sample/pom.xml
@@ -5,7 +5,7 @@
hadoop-unit
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
pom
@@ -16,6 +16,7 @@
parquet-spark
kafka-stream
spark-streaming-cassandra
+ all
sample
diff --git a/sample/solr-parquet-spark/pom.xml b/sample/solr-parquet-spark/pom.xml
index 0f8b8acb..9e00e1f3 100644
--- a/sample/solr-parquet-spark/pom.xml
+++ b/sample/solr-parquet-spark/pom.xml
@@ -5,13 +5,12 @@
sample
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
solr-parquet-spark
-
org.slf4j
@@ -48,23 +47,112 @@
com.lucidworks.spark
spark-solr
+
+
+ log4j
+ log4j
+
+
+
+
+
+ com.databricks
+ spark-csv_2.10
+ 1.5.0
fr.jetoile.hadoop
hadoop-unit-commons
test
+
+
+ log4j
+ log4j
+
+
fr.jetoile.hadoop
hadoop-unit-client-hdfs
test
+
+
+ log4j
+ log4j
+
+
-
- fr.jetoile.hadoop
- hadoop-unit-client-hive
-
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ **/*IntegrationTest.java
+
+
+
+
+ integration-test
+
+ test
+
+ integration-test
+
+
+ none
+
+
+ **/*IntegrationTest.java
+
+
+
+
+
+
+
+ hadoop-unit-maven-plugin
+ fr.jetoile.hadoop
+ ${hadoop-unit.version}
+
+
+ start
+
+ embedded-start
+
+ pre-integration-test
+
+
+
+ /home/khanh/.m2/repository
+ https://repo.maven.apache.org/maven2/
+
+
+ HDFS
+ fr.jetoile.hadoop:hadoop-unit-hdfs:${hadoop-unit.version}
+
+
+ ZOOKEEPER
+ fr.jetoile.hadoop:hadoop-unit-zookeeper:${hadoop-unit.version}
+
+
+ SOLRCLOUD
+ fr.jetoile.hadoop:hadoop-unit-solrcloud:${hadoop-unit.version}
+
+ file://${project.basedir}/src/test/resources/solr
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/sample/solr-parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/ParquetToSolrJobTest.java b/sample/solr-parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/ParquetToSolrJobIntegrationTest.java
similarity index 62%
rename from sample/solr-parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/ParquetToSolrJobTest.java
rename to sample/solr-parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/ParquetToSolrJobIntegrationTest.java
index af6dc519..96d24f65 100644
--- a/sample/solr-parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/ParquetToSolrJobTest.java
+++ b/sample/solr-parquet-spark/src/test/java/fr/jetoile/hadoopunit/sample/ParquetToSolrJobIntegrationTest.java
@@ -14,14 +14,10 @@
package fr.jetoile.hadoopunit.sample;
-import com.ninja_squad.dbsetup.Operations;
-import com.ninja_squad.dbsetup.operation.Operation;
import fr.jetoile.hadoopunit.HadoopUnitConfig;
import fr.jetoile.hadoopunit.exception.BootstrapException;
import fr.jetoile.hadoopunit.exception.NotFoundServiceException;
import fr.jetoile.hadoopunit.test.hdfs.HdfsUtils;
-import fr.jetoile.hadoopunit.test.hive.HiveConnectionUtils;
-import fr.jetoile.hadoopunit.test.hive.HiveSetup;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
@@ -33,8 +29,11 @@
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.hive.HiveContext;
-import org.junit.*;
+import org.apache.spark.sql.SQLContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,22 +41,16 @@
import java.net.URISyntaxException;
import java.sql.SQLException;
-import static com.ninja_squad.dbsetup.Operations.sequenceOf;
-import static com.ninja_squad.dbsetup.Operations.sql;
import static junit.framework.TestCase.assertNotNull;
import static org.fest.assertions.Assertions.assertThat;
-@Ignore
-public class ParquetToSolrJobTest {
+public class ParquetToSolrJobIntegrationTest {
- static private Logger LOGGER = LoggerFactory.getLogger(ParquetToSolrJobTest.class);
+ static private Logger LOGGER = LoggerFactory.getLogger(ParquetToSolrJobIntegrationTest.class);
private static Configuration configuration;
- public static Operation CREATE_TABLES = null;
- public static Operation DROP_TABLES = null;
-
@BeforeClass
public static void setUp() throws BootstrapException, SQLException, ClassNotFoundException, NotFoundServiceException {
@@ -67,19 +60,6 @@ public static void setUp() throws BootstrapException, SQLException, ClassNotFoun
throw new BootstrapException("bad config", e);
}
- CREATE_TABLES =
- sequenceOf(sql("CREATE EXTERNAL TABLE IF NOT EXISTS default.test(id INT, value STRING) " +
- " ROW FORMAT DELIMITED FIELDS TERMINATED BY ';'" +
- " STORED AS TEXTFILE" +
- " LOCATION 'hdfs://localhost:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY) + "/khanh/test'"),
- sql("CREATE EXTERNAL TABLE IF NOT EXISTS default.test_parquet(id INT, value STRING) " +
- " ROW FORMAT DELIMITED FIELDS TERMINATED BY ';'" +
- " STORED AS PARQUET" +
- " LOCATION 'hdfs://localhost:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY) + "/khanh/test_parquet'"));
-
- DROP_TABLES =
- sequenceOf(sql("DROP TABLE IF EXISTS default.test"),
- sql("DROP TABLE IF EXISTS default.test_parquet"));
}
@Before
@@ -88,15 +68,11 @@ public void before() throws IOException, URISyntaxException {
fileSystem.mkdirs(new Path("/khanh/test"));
fileSystem.mkdirs(new Path("/khanh/test_parquet"));
- fileSystem.copyFromLocalFile(new Path(ParquetToSolrJobTest.class.getClassLoader().getResource("test.csv").toURI()), new Path("/khanh/test/test.csv"));
-
- new HiveSetup(HiveConnectionUtils.INSTANCE.getDestination(), Operations.sequenceOf(CREATE_TABLES)).launch();
+ fileSystem.copyFromLocalFile(new Path(ParquetToSolrJobIntegrationTest.class.getClassLoader().getResource("test.csv").toURI()), new Path("/khanh/test/test.csv"));
}
@After
public void clean() throws IOException {
- new HiveSetup(HiveConnectionUtils.INSTANCE.getDestination(), Operations.sequenceOf(DROP_TABLES)).launch();
-
FileSystem fileSystem = HdfsUtils.INSTANCE.getFileSystem();
fileSystem.delete(new Path("/khanh"), true);
}
@@ -110,12 +86,14 @@ public void spark_should_read_parquet_file_and_index_into_solr() throws IOExcept
.setAppName("test");
JavaSparkContext context = new JavaSparkContext(conf);
-
- //read hive-site from classpath
- HiveContext hiveContext = new HiveContext(context.sc());
-
- DataFrame sql = hiveContext.sql("SELECT * FROM default.test");
- sql.write().parquet("hdfs://localhost:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY) + "/khanh/test_parquet/file.parquet");
+ SQLContext sqlContext = new SQLContext(context);
+ DataFrame df = sqlContext.read()
+ .format("com.databricks.spark.csv")
+ .option("header", "true") // Use first line of all files as header
+ .option("inferSchema", "true") // Automatically infer data types
+ .load("hdfs://localhost:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY) + "/khanh/test/test.csv");
+
+ df.write().parquet("hdfs://localhost:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY) + "/khanh/test_parquet/file.parquet");
FileSystem fileSystem = HdfsUtils.INSTANCE.getFileSystem();
assertThat(fileSystem.exists(new Path("hdfs://localhost:" + configuration.getInt(HadoopUnitConfig.HDFS_NAMENODE_PORT_KEY) + "/khanh/test_parquet/file.parquet"))).isTrue();
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/currency.xml b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/currency.xml
new file mode 100644
index 00000000..3a9c58af
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/currency.xml
@@ -0,0 +1,67 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt
new file mode 100644
index 00000000..2c164c0b
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# a couple of test stopwords to test that the words are really being
+# configured from this file:
+stopworda
+stopwordb
+
+# Standard english stop words taken from Lucene's StopAnalyzer
+a
+an
+and
+are
+as
+at
+be
+but
+by
+for
+if
+in
+into
+is
+it
+no
+not
+of
+on
+or
+such
+that
+the
+their
+then
+there
+these
+they
+this
+to
+was
+will
+with
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/protwords.txt b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/protwords.txt
new file mode 100644
index 00000000..1dfc0abe
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/protwords.txt
@@ -0,0 +1,21 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# Use a protected word file to protect against the stemmer reducing two
+# unrelated words to the same base word.
+
+# Some non-words that normally won't be encountered,
+# just to test that they won't be stemmed.
+dontstems
+zwhacky
+
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/schema.xml b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/schema.xml
new file mode 100644
index 00000000..a5083977
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/schema.xml
@@ -0,0 +1,748 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ id
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/solrconfig.xml b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/solrconfig.xml
new file mode 100644
index 00000000..3de5587f
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/solrconfig.xml
@@ -0,0 +1,1823 @@
+
+
+
+
+
+
+
+
+ 5.3.0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.data.dir:}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.lock.type:none}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.ulog.dir:}
+
+
+
+
+ ${solr.autoCommit.maxTime:15000}
+ false
+
+
+
+
+
+ ${solr.autoSoftCommit.maxTime:-1}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1024
+
+
+
+ -1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+ 20
+
+
+ 200
+
+
+
+
+
+
+
+
+
+
+
+ static firstSearcher warming in solrconfig.xml
+
+
+
+
+
+ false
+
+
+ 2
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ explicit
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+ explicit
+ 10
+ text
+
+
+ inStock:true
+
+
+
+
+
+
+ explicit
+ json
+ true
+ text
+
+
+
+
+
+
+
+
+
+ {!xport}
+ xsort
+ false
+
+
+
+ query
+
+
+
+
+
+
+
+
+
+
+
+ explicit
+
+
+ velocity
+ browse
+ layout
+ Solritas
+
+
+ edismax
+
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+
+ 100%
+ *:*
+ 10
+ *,score
+
+
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+
+ text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+ 3
+
+
+ on
+ true
+ cat
+ manu_exact
+ content_type
+ author_s
+ ipod
+ GB
+ 1
+ cat,inStock
+ after
+ price
+ 0
+ 600
+ 50
+ popularity
+ 0
+ 10
+ 3
+ manufacturedate_dt
+ NOW/YEAR-10YEARS
+ NOW
+ +1YEAR
+ before
+ after
+
+
+ on
+ content features title name
+ true
+ html
+ <b>
+ </b>
+ 0
+ title
+ 0
+ name
+ 3
+ 200
+ content
+ 750
+
+
+ on
+ false
+ 5
+ 2
+ 5
+ true
+ true
+ 5
+ 3
+
+
+
+
+ spellcheck
+
+
+
+
+
+
+
+
+ text
+
+
+
+
+
+
+ _src_
+
+ true
+
+
+
+
+
+
+
+
+
+ true
+ ignored_
+
+
+ true
+ links
+ ignored_
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ explicit
+ true
+
+
+
+
+
+
+
+
+ text_general
+
+
+
+
+
+ default
+ text
+ solr.DirectSolrSpellChecker
+
+ internal
+
+ 0.5
+
+ 2
+
+ 1
+
+ 5
+
+ 4
+
+ 0.01
+
+
+
+
+
+ wordbreak
+ solr.WordBreakSolrSpellChecker
+ name
+ true
+ true
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ default
+ wordbreak
+ on
+ true
+ 10
+ 5
+ 5
+ true
+ true
+ 10
+ 5
+
+
+ spellcheck
+
+
+
+
+
+
+ mySuggester
+ FuzzyLookupFactory
+ DocumentDictionaryFactory
+ cat
+ price
+ string
+
+
+
+
+
+ true
+ 10
+
+
+ suggest
+
+
+ -->
+
+
+
+
+
+
+
+
+ true
+
+
+ tvComponent
+
+
+
+
+
+
+ lingo
+
+
+ org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+
+
+ clustering/carrot2
+
+
+
+
+ stc
+ org.carrot2.clustering.stc.STCClusteringAlgorithm
+
+
+
+
+ kmeans
+ org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+
+
+
+
+
+
+ true
+ true
+
+ name
+
+ id
+
+ features
+
+ true
+
+
+
+ false
+
+
+ edismax
+
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+
+ *:*
+ 10
+ *,score
+
+
+ clustering
+
+
+
+
+
+
+
+
+
+ true
+ false
+
+
+ terms
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 100
+
+
+
+
+
+
+
+ 70
+
+ 0.5
+
+ [-\w ,/\n\"']{20,200}
+
+
+
+
+
+
+ ]]>
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ,,
+ ,,
+ ,,
+ ,,
+ ,]]>
+ ]]>
+
+
+
+
+
+ 10
+ .,!?
+
+
+
+
+
+
+ WORD
+
+
+ en
+ US
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${velocity.template.base.dir:}
+
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ *:*
+
+
+
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/stopwords.txt b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/stopwords.txt
new file mode 100644
index 00000000..ae1e83ee
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/stopwords.txt
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/synonyms.txt b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/synonyms.txt
new file mode 100644
index 00000000..7f721283
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/conf/synonyms.txt
@@ -0,0 +1,29 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+#some test synonym mappings unlikely to appear in real input text
+aaafoo => aaabar
+bbbfoo => bbbfoo bbbbar
+cccfoo => cccbar cccbaz
+fooaaa,baraaa,bazaaa
+
+# Some synonym groups specific to this example
+GB,gib,gigabyte,gigabytes
+MB,mib,megabyte,megabytes
+Television, Televisions, TV, TVs
+#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
+#after us won't split it into two words.
+
+# Synonym mappings can be used for spelling correction too
+pixima => pixma
+
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/collection1/core.properties b/sample/solr-parquet-spark/src/test/resources/solr/collection1/core.properties
new file mode 100644
index 00000000..2d3491d9
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/collection1/core.properties
@@ -0,0 +1,5 @@
+numShards=1
+name=collection1_shard1_replica1
+shard=shard1
+collection=collection1
+coreNodeName=core_node1
diff --git a/sample/solr-parquet-spark/src/test/resources/solr/solr.xml b/sample/solr-parquet-spark/src/test/resources/solr/solr.xml
new file mode 100644
index 00000000..6e835c9d
--- /dev/null
+++ b/sample/solr-parquet-spark/src/test/resources/solr/solr.xml
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+ ${host:}
+ ${jetty.port:8983}
+ ${hostContext:solr}
+ ${zkClientTimeout:15000}
+ ${genericCoreNodeNames:true}
+
+
+
+ ${socketTimeout:0}
+ ${connTimeout:0}
+
+
+
\ No newline at end of file
diff --git a/sample/solr-parquet-spark/src/test/resources/test.csv b/sample/solr-parquet-spark/src/test/resources/test.csv
index 47ede0f1..7ebf9617 100644
--- a/sample/solr-parquet-spark/src/test/resources/test.csv
+++ b/sample/solr-parquet-spark/src/test/resources/test.csv
@@ -1,3 +1,4 @@
-1;value1
-2;value2
-3;value3
\ No newline at end of file
+id,value
+1,value1
+2,value2
+3,value3
\ No newline at end of file
diff --git a/sample/spark-streaming-cassandra/pom.xml b/sample/spark-streaming-cassandra/pom.xml
index f30e2613..08b13ea9 100644
--- a/sample/spark-streaming-cassandra/pom.xml
+++ b/sample/spark-streaming-cassandra/pom.xml
@@ -5,7 +5,7 @@
sample
fr.jetoile.hadoop
- 1.6-SNAPSHOT
+ 2.0-SNAPSHOT
4.0.0
@@ -91,4 +91,70 @@
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ **/*IntegrationTest.java
+
+
+
+
+ integration-test
+
+ test
+
+ integration-test
+
+
+ none
+
+
+ **/*IntegrationTest.java
+
+
+
+
+
+
+
+ hadoop-unit-maven-plugin
+ fr.jetoile.hadoop
+ ${hadoop-unit.version}
+
+
+ start
+
+ embedded-start
+
+ pre-integration-test
+
+
+
+ /home/khanh/.m2/repository
+ https://repo.maven.apache.org/maven2/
+
+
+ ZOOKEEPER
+ fr.jetoile.hadoop:hadoop-unit-zookeeper:${hadoop-unit.version}
+
+
+ KAFKA
+ fr.jetoile.hadoop:hadoop-unit-kafka:${hadoop-unit.version}
+
+
+ CASSANDRA
+ fr.jetoile.hadoop:hadoop-unit-cassandra:${hadoop-unit.version}
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/sample/spark-streaming-cassandra/src/test/java/fr/jetoile/hadoopunit/sample/SparkKafkaIntegrationTest.java b/sample/spark-streaming-cassandra/src/test/java/fr/jetoile/hadoopunit/sample/SparkKafkaIntegrationIntegrationTest.java
similarity index 98%
rename from sample/spark-streaming-cassandra/src/test/java/fr/jetoile/hadoopunit/sample/SparkKafkaIntegrationTest.java
rename to sample/spark-streaming-cassandra/src/test/java/fr/jetoile/hadoopunit/sample/SparkKafkaIntegrationIntegrationTest.java
index 34d8bc50..63c4a567 100644
--- a/sample/spark-streaming-cassandra/src/test/java/fr/jetoile/hadoopunit/sample/SparkKafkaIntegrationTest.java
+++ b/sample/spark-streaming-cassandra/src/test/java/fr/jetoile/hadoopunit/sample/SparkKafkaIntegrationIntegrationTest.java
@@ -23,12 +23,11 @@
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
-@Ignore
-public class SparkKafkaIntegrationTest {
+public class SparkKafkaIntegrationIntegrationTest {
private static Configuration configuration;
- private static final Logger LOGGER = LoggerFactory.getLogger(SparkKafkaIntegrationTest.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(SparkKafkaIntegrationIntegrationTest.class);
private static final FastDateFormat DATE_FORMATTER = FastDateFormat.getInstance("MM/dd/yyyy HH:mm:ss.SSS");