add mongodb support

hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java

@@ -26,7 +26,8 @@ public enum Component {
     HBASE("hbase"),
     OOZIE("oozie"),
     SOLRCLOUD("solrcloud"),
-    SOLR("solr");
+    SOLR("solr"),
+    MONGODB("mongodb");
 
     private String key;
 

hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Config.java

@@ -41,6 +41,12 @@ public class Config {
     public static final String HIVE_TEST_DATABASE_NAME_KEY = "hive.test.database.name";
     public static final String HIVE_TEST_TABLE_NAME_KEY = "hive.test.table.name";
 
+    // MongoDB
+    public static final String MONGO_IP_KEY = "mongo.ip";
+    public static final String MONGO_PORT_KEY = "mongo.port";
+    public static final String MONGO_DATABASE_NAME_KEY = "mongo.database.name";
+    public static final String MONGO_COLLECTION_NAME_KEY = "mongo.collection.name";
+
     // Kafka
     public static final String KAFKA_HOSTNAME_KEY = "kafka.hostname";
     public static final String KAFKA_PORT_KEY = "kafka.port";

hadoop-unit-mongodb/pom.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>hadoop-unit</artifactId>
+        <groupId>fr.jetoile.hadoop</groupId>
+        <version>1.2-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>hadoop-unit-mongodb</artifactId>
+
+    <properties>
+        <mongo-java-driver.version>2.11.3</mongo-java-driver.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>fr.jetoile.hadoop</groupId>
+            <artifactId>hadoop-unit-commons</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.github.sakserv</groupId>
+            <artifactId>hadoop-mini-clusters-mongodb</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>mongo-java-driver</artifactId>
+            <version>${mongo-java-driver.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+
+</project>

hadoop-unit-mongodb/src/main/java/fr/jetoile/hadoopunit/component/MongoDbBootstrap.java

@@ -0,0 +1,122 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.jetoile.hadoopunit.component;
+
+import com.github.sakserv.minicluster.impl.MongodbLocalServer;
+import fr.jetoile.hadoopunit.Component;
+import fr.jetoile.hadoopunit.Config;
+import fr.jetoile.hadoopunit.exception.BootstrapException;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MongoDbBootstrap implements Bootstrap {
+    final public static String NAME = Component.MONGODB.name();
+
+    final private Logger LOGGER = LoggerFactory.getLogger(MongoDbBootstrap.class);
+
+    private State state = State.STOPPED;
+
+    private Configuration configuration;
+
+    private MongodbLocalServer mongodbLocalServer;
+
+    private int port;
+    private String ip;
+
+    public MongoDbBootstrap() {
+        if (mongodbLocalServer == null) {
+            try {
+                loadConfig();
+            } catch (BootstrapException e) {
+                LOGGER.error("unable to load configuration", e);
+            }
+        }
+    }
+
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public String getProperties() {
+        return "[" +
+                "ip:" + ip +
+                ", port:" + port +
+                "]";
+    }
+
+    private void loadConfig() throws BootstrapException {
+        try {
+            configuration = new PropertiesConfiguration(Config.DEFAULT_PROPS_FILE);
+        } catch (ConfigurationException e) {
+            throw new BootstrapException("bad config", e);
+        }
+
+        port = configuration.getInt(Config.MONGO_PORT_KEY);
+        ip = configuration.getString(Config.MONGO_IP_KEY);
+    }
+
+    private void build() {
+        mongodbLocalServer = new MongodbLocalServer.Builder()
+                .setIp(ip)
+                .setPort(port)
+                .build();
+    }
+
+    @Override
+    public Bootstrap start() {
+        if (state == State.STOPPED) {
+            state = State.STARTING;
+            LOGGER.info("{} is starting", this.getClass().getName());
+            build();
+            try {
+                mongodbLocalServer.start();
+            } catch (Exception e) {
+                LOGGER.error("unable to add mongodb", e);
+            }
+            state = State.STARTED;
+            LOGGER.info("{} is started", this.getClass().getName());
+        }
+
+        return this;
+    }
+
+    @Override
+    public Bootstrap stop() {
+        if (state == State.STARTED) {
+            state = State.STOPPING;
+            LOGGER.info("{} is stopping", this.getClass().getName());
+            try {
+                mongodbLocalServer.stop(true);
+            } catch (Exception e) {
+                LOGGER.error("unable to stop mongdb", e);
+            }
+            state = State.STOPPED;
+            LOGGER.info("{} is stopped", this.getClass().getName());
+        }
+        return this;
+    }
+
+    @Override
+    public org.apache.hadoop.conf.Configuration getConfiguration() {
+        throw new UnsupportedOperationException("the method getConfiguration can not be called on MongoDbBootstrap");
+    }
+
+}

hadoop-unit-mongodb/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap

@@ -0,0 +1 @@
+fr.jetoile.hadoopunit.component.MongoDbBootstrap

hadoop-unit-mongodb/src/test/java/fr/jetoile/hadoopunit/component/MongoDbBootstrapTest.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.jetoile.hadoopunit.component;
+
+import com.mongodb.*;
+import fr.jetoile.hadoopunit.Config;
+import fr.jetoile.hadoopunit.HadoopBootstrap;
+import fr.jetoile.hadoopunit.exception.BootstrapException;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.UnknownHostException;
+import java.util.Date;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class MongoDbBootstrapTest {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MongoDbBootstrapTest.class);
+
+    static private Configuration configuration;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        HadoopBootstrap.INSTANCE.startAll();
+
+        try {
+            configuration = new PropertiesConfiguration(Config.DEFAULT_PROPS_FILE);
+        } catch (ConfigurationException e) {
+            throw new BootstrapException("bad config", e);
+        }
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        HadoopBootstrap.INSTANCE.stopAll();
+    }
+
+    @Test
+    public void mongodbShouldStart() throws UnknownHostException {
+        MongoClient mongo = new MongoClient(configuration.getString(Config.MONGO_IP_KEY), configuration.getInt(Config.MONGO_PORT_KEY));
+
+        DB db = mongo.getDB(configuration.getString(Config.MONGO_DATABASE_NAME_KEY));
+        DBCollection col = db.createCollection(configuration.getString(Config.MONGO_COLLECTION_NAME_KEY),
+                new BasicDBObject());
+
+        col.save(new BasicDBObject("testDoc", new Date()));
+        LOG.info("MONGODB: Number of items in collection: {}", col.count());
+        assertEquals(1, col.count());
+
+        DBCursor cursor = col.find();
+        while(cursor.hasNext()) {
+            LOG.info("MONGODB: Document output: {}", cursor.next());
+        }
+        cursor.close();
+    }
+}

hadoop-unit-mongodb/src/test/resources/default.properties

@@ -0,0 +1,121 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#HADOOP_HOME=/opt/hadoop
+
+# Zookeeper
+zookeeper.temp.dir=/tmp/embedded_zk
+zookeeper.host=127.0.0.1
+zookeeper.port=22010
+
+# Hive
+hive.scratch.dir=/tmp/hive_scratch_dir
+hive.warehouse.dir=/tmp/warehouse_dir
+
+# Hive Metastore
+hive.metastore.hostname=localhost
+hive.metastore.port=20102
+hive.metastore.derby.db.dir=metastore_db
+
+# Hive Server2
+hive.server2.hostname=localhost
+hive.server2.port=20103
+
+# Hive Test
+hive.test.database.name=default
+hive.test.table.name=test_table
+
+
+# HDFS
+hdfs.namenode.port=20112
+hdfs.namenode.http.port=50070
+hdfs.temp.dir=/tmp/embedded_hdfs
+hdfs.num.datanodes=1
+hdfs.enable.permissions=false
+hdfs.format=true
+hdfs.enable.running.user.as.proxy.user=true
+
+# HDFS Test
+hdfs.test.file=/tmp/testing
+hdfs.test.string=TESTING
+
+
+# HBase
+hbase.master.port=25111
+hbase.master.info.port=-1
+hbase.num.region.servers=1
+hbase.root.dir=/tmp/embedded_hbase
+hbase.znode.parent=/hbase-unsecure
+hbase.wal.replication.enabled=false
+
+# HBase Test
+hbase.test.table.name=hbase_test_table
+hbase.test.col.family.name=cf1
+hbase.test.col.qualifier.name=cq1
+hbase.test.num.rows.to.put=50
+
+# Kafka
+kafka.hostname=127.0.0.1
+kafka.port=20111
+
+# Kafka Test
+kafka.test.topic=testtopic
+kafka.test.message.count=10
+kafka.test.broker.id=1
+kafka.test.temp.dir=embedded_kafka
+
+#SolR + SolRCloud
+solr.dir=solr
+
+#SolR
+solr.collection.internal.name=collection1_shard1_replica1
+
+#SolRCloud
+solr.collection.name=collection1
+solr.cloud.port=8983
+
+
+# YARN
+yarn.num.node.managers=1
+yarn.num.local.dirs=1
+yarn.num.log.dirs=1
+yarn.resource.manager.address=localhost:37001
+yarn.resource.manager.hostname=localhost
+yarn.resource.manager.scheduler.address=localhost:37002
+yarn.resource.manager.resource.tracker.address=localhost:37003
+yarn.resource.manager.webapp.address=localhost:37004
+yarn.use.in.jvm.container.executor=false
+
+# MR
+mr.job.history.address=localhost:37005
+
+# Oozie
+oozie.test.dir=/tmp/embedded_oozie
+oozie.home.dir=/tmp/oozie_home
+oozie.username=blah
+oozie.groupname=testgroup
+oozie.hdfs.share.lib.dir=/tmp/share_lib
+oozie.share.lib.create=true
+oozie.local.share.lib.cache.dir=/tmp/share_lib_cache
+oozie.purge.local.share.lib.cache=false
+oozie.sharelib.path=/home/khanh/github
+oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz
+oozie.port=20113
+oozie.host=localhost
+
+# MongoDB
+mongo.ip=127.0.0.1
+mongo.port=13333
+mongo.database.name=test_database
+mongo.collection.name=test_collection