From 6034598fca16b3f6192185407af0232a35762c0c Mon Sep 17 00:00:00 2001 From: jetoile Date: Mon, 18 Apr 2016 12:25:33 +0200 Subject: [PATCH] add mongodb support --- .../java/fr/jetoile/hadoopunit/Component.java | 3 +- .../java/fr/jetoile/hadoopunit/Config.java | 6 + hadoop-unit-mongodb/pom.xml | 39 ++++++ .../component/MongoDbBootstrap.java | 122 ++++++++++++++++++ .../fr.jetoile.hadoopunit.component.Bootstrap | 1 + .../component/MongoDbBootstrapTest.java | 76 +++++++++++ .../src/test/resources/default.properties | 121 +++++++++++++++++ hadoop-unit-standalone/pom.xml | 17 +++ .../src/main/conf/default.properties | 8 +- .../src/main/conf/hadoop.properties | 1 + .../src/main/resources/default.properties | 8 +- .../src/main/resources/hadoop.properties | 1 + .../IntegrationBootstrapTest.java | 10 +- .../src/test/resources/default.properties | 13 +- pom.xml | 7 + 15 files changed, 425 insertions(+), 8 deletions(-) create mode 100644 hadoop-unit-mongodb/pom.xml create mode 100644 hadoop-unit-mongodb/src/main/java/fr/jetoile/hadoopunit/component/MongoDbBootstrap.java create mode 100644 hadoop-unit-mongodb/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap create mode 100644 hadoop-unit-mongodb/src/test/java/fr/jetoile/hadoopunit/component/MongoDbBootstrapTest.java create mode 100644 hadoop-unit-mongodb/src/test/resources/default.properties diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java index 13db3e5d..3278c90c 100644 --- a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java +++ b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Component.java @@ -26,7 +26,8 @@ public enum Component { HBASE("hbase"), OOZIE("oozie"), SOLRCLOUD("solrcloud"), - SOLR("solr"); + SOLR("solr"), + MONGODB("mongodb"); private String key; diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Config.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Config.java index 4b5d16c4..e30a2d6e 100644 --- a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Config.java +++ b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/Config.java @@ -41,6 +41,12 @@ public class Config { public static final String HIVE_TEST_DATABASE_NAME_KEY = "hive.test.database.name"; public static final String HIVE_TEST_TABLE_NAME_KEY = "hive.test.table.name"; + // MongoDB + public static final String MONGO_IP_KEY = "mongo.ip"; + public static final String MONGO_PORT_KEY = "mongo.port"; + public static final String MONGO_DATABASE_NAME_KEY = "mongo.database.name"; + public static final String MONGO_COLLECTION_NAME_KEY = "mongo.collection.name"; + // Kafka public static final String KAFKA_HOSTNAME_KEY = "kafka.hostname"; public static final String KAFKA_PORT_KEY = "kafka.port"; diff --git a/hadoop-unit-mongodb/pom.xml b/hadoop-unit-mongodb/pom.xml new file mode 100644 index 00000000..e1669499 --- /dev/null +++ b/hadoop-unit-mongodb/pom.xml @@ -0,0 +1,39 @@ + + + + + hadoop-unit + fr.jetoile.hadoop + 1.2-SNAPSHOT + + 4.0.0 + + hadoop-unit-mongodb + + + 2.11.3 + + + + + fr.jetoile.hadoop + hadoop-unit-commons + + + + com.github.sakserv + hadoop-mini-clusters-mongodb + + + + org.mongodb + mongo-java-driver + ${mongo-java-driver.version} + test + + + + + \ No newline at end of file diff --git a/hadoop-unit-mongodb/src/main/java/fr/jetoile/hadoopunit/component/MongoDbBootstrap.java b/hadoop-unit-mongodb/src/main/java/fr/jetoile/hadoopunit/component/MongoDbBootstrap.java new file mode 100644 index 00000000..15714571 --- /dev/null +++ b/hadoop-unit-mongodb/src/main/java/fr/jetoile/hadoopunit/component/MongoDbBootstrap.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fr.jetoile.hadoopunit.component; + +import com.github.sakserv.minicluster.impl.MongodbLocalServer; +import fr.jetoile.hadoopunit.Component; +import fr.jetoile.hadoopunit.Config; +import fr.jetoile.hadoopunit.exception.BootstrapException; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoDbBootstrap implements Bootstrap { + final public static String NAME = Component.MONGODB.name(); + + final private Logger LOGGER = LoggerFactory.getLogger(MongoDbBootstrap.class); + + private State state = State.STOPPED; + + private Configuration configuration; + + private MongodbLocalServer mongodbLocalServer; + + private int port; + private String ip; + + public MongoDbBootstrap() { + if (mongodbLocalServer == null) { + try { + loadConfig(); + } catch (BootstrapException e) { + LOGGER.error("unable to load configuration", e); + } + } + } + + + @Override + public String getName() { + return NAME; + } + + @Override + public String getProperties() { + return "[" + + "ip:" + ip + + ", port:" + port + + "]"; + } + + private void loadConfig() throws BootstrapException { + try { + configuration = new PropertiesConfiguration(Config.DEFAULT_PROPS_FILE); + } catch (ConfigurationException e) { + throw new BootstrapException("bad config", e); + } + + port = configuration.getInt(Config.MONGO_PORT_KEY); + ip = configuration.getString(Config.MONGO_IP_KEY); + } + + private void build() { + mongodbLocalServer = new MongodbLocalServer.Builder() + .setIp(ip) + .setPort(port) + .build(); + } + + @Override + public Bootstrap start() { + if (state == State.STOPPED) { + state = State.STARTING; + LOGGER.info("{} is starting", this.getClass().getName()); + build(); + try { + mongodbLocalServer.start(); + } catch (Exception e) { + LOGGER.error("unable to add mongodb", e); + } + state = State.STARTED; + LOGGER.info("{} is started", this.getClass().getName()); + } + + return this; + } + + @Override + public Bootstrap stop() { + if (state == State.STARTED) { + state = State.STOPPING; + LOGGER.info("{} is stopping", this.getClass().getName()); + try { + mongodbLocalServer.stop(true); + } catch (Exception e) { + LOGGER.error("unable to stop mongdb", e); + } + state = State.STOPPED; + LOGGER.info("{} is stopped", this.getClass().getName()); + } + return this; + } + + @Override + public org.apache.hadoop.conf.Configuration getConfiguration() { + throw new UnsupportedOperationException("the method getConfiguration can not be called on MongoDbBootstrap"); + } + +} diff --git a/hadoop-unit-mongodb/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap b/hadoop-unit-mongodb/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap new file mode 100644 index 00000000..f185db38 --- /dev/null +++ b/hadoop-unit-mongodb/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap @@ -0,0 +1 @@ +fr.jetoile.hadoopunit.component.MongoDbBootstrap \ No newline at end of file diff --git a/hadoop-unit-mongodb/src/test/java/fr/jetoile/hadoopunit/component/MongoDbBootstrapTest.java b/hadoop-unit-mongodb/src/test/java/fr/jetoile/hadoopunit/component/MongoDbBootstrapTest.java new file mode 100644 index 00000000..ee8cb051 --- /dev/null +++ b/hadoop-unit-mongodb/src/test/java/fr/jetoile/hadoopunit/component/MongoDbBootstrapTest.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fr.jetoile.hadoopunit.component; + +import com.mongodb.*; +import fr.jetoile.hadoopunit.Config; +import fr.jetoile.hadoopunit.HadoopBootstrap; +import fr.jetoile.hadoopunit.exception.BootstrapException; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.UnknownHostException; +import java.util.Date; + +import static org.junit.Assert.assertEquals; + + +public class MongoDbBootstrapTest { + + private static final Logger LOG = LoggerFactory.getLogger(MongoDbBootstrapTest.class); + + static private Configuration configuration; + + @BeforeClass + public static void setUp() throws Exception { + HadoopBootstrap.INSTANCE.startAll(); + + try { + configuration = new PropertiesConfiguration(Config.DEFAULT_PROPS_FILE); + } catch (ConfigurationException e) { + throw new BootstrapException("bad config", e); + } + } + + @AfterClass + public static void tearDown() throws Exception { + HadoopBootstrap.INSTANCE.stopAll(); + } + + @Test + public void mongodbShouldStart() throws UnknownHostException { + MongoClient mongo = new MongoClient(configuration.getString(Config.MONGO_IP_KEY), configuration.getInt(Config.MONGO_PORT_KEY)); + + DB db = mongo.getDB(configuration.getString(Config.MONGO_DATABASE_NAME_KEY)); + DBCollection col = db.createCollection(configuration.getString(Config.MONGO_COLLECTION_NAME_KEY), + new BasicDBObject()); + + col.save(new BasicDBObject("testDoc", new Date())); + LOG.info("MONGODB: Number of items in collection: {}", col.count()); + assertEquals(1, col.count()); + + DBCursor cursor = col.find(); + while(cursor.hasNext()) { + LOG.info("MONGODB: Document output: {}", cursor.next()); + } + cursor.close(); + } +} \ No newline at end of file diff --git a/hadoop-unit-mongodb/src/test/resources/default.properties b/hadoop-unit-mongodb/src/test/resources/default.properties new file mode 100644 index 00000000..4579cb8b --- /dev/null +++ b/hadoop-unit-mongodb/src/test/resources/default.properties @@ -0,0 +1,121 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#HADOOP_HOME=/opt/hadoop + +# Zookeeper +zookeeper.temp.dir=/tmp/embedded_zk +zookeeper.host=127.0.0.1 +zookeeper.port=22010 + +# Hive +hive.scratch.dir=/tmp/hive_scratch_dir +hive.warehouse.dir=/tmp/warehouse_dir + +# Hive Metastore +hive.metastore.hostname=localhost +hive.metastore.port=20102 +hive.metastore.derby.db.dir=metastore_db + +# Hive Server2 +hive.server2.hostname=localhost +hive.server2.port=20103 + +# Hive Test +hive.test.database.name=default +hive.test.table.name=test_table + + +# HDFS +hdfs.namenode.port=20112 +hdfs.namenode.http.port=50070 +hdfs.temp.dir=/tmp/embedded_hdfs +hdfs.num.datanodes=1 +hdfs.enable.permissions=false +hdfs.format=true +hdfs.enable.running.user.as.proxy.user=true + +# HDFS Test +hdfs.test.file=/tmp/testing +hdfs.test.string=TESTING + + +# HBase +hbase.master.port=25111 +hbase.master.info.port=-1 +hbase.num.region.servers=1 +hbase.root.dir=/tmp/embedded_hbase +hbase.znode.parent=/hbase-unsecure +hbase.wal.replication.enabled=false + +# HBase Test +hbase.test.table.name=hbase_test_table +hbase.test.col.family.name=cf1 +hbase.test.col.qualifier.name=cq1 +hbase.test.num.rows.to.put=50 + +# Kafka +kafka.hostname=127.0.0.1 +kafka.port=20111 + +# Kafka Test +kafka.test.topic=testtopic +kafka.test.message.count=10 +kafka.test.broker.id=1 +kafka.test.temp.dir=embedded_kafka + +#SolR + SolRCloud +solr.dir=solr + +#SolR +solr.collection.internal.name=collection1_shard1_replica1 + +#SolRCloud +solr.collection.name=collection1 +solr.cloud.port=8983 + + +# YARN +yarn.num.node.managers=1 +yarn.num.local.dirs=1 +yarn.num.log.dirs=1 +yarn.resource.manager.address=localhost:37001 +yarn.resource.manager.hostname=localhost +yarn.resource.manager.scheduler.address=localhost:37002 +yarn.resource.manager.resource.tracker.address=localhost:37003 +yarn.resource.manager.webapp.address=localhost:37004 +yarn.use.in.jvm.container.executor=false + +# MR +mr.job.history.address=localhost:37005 + +# Oozie +oozie.test.dir=/tmp/embedded_oozie +oozie.home.dir=/tmp/oozie_home +oozie.username=blah +oozie.groupname=testgroup +oozie.hdfs.share.lib.dir=/tmp/share_lib +oozie.share.lib.create=true +oozie.local.share.lib.cache.dir=/tmp/share_lib_cache +oozie.purge.local.share.lib.cache=false +oozie.sharelib.path=/home/khanh/github +oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz +oozie.port=20113 +oozie.host=localhost + +# MongoDB +mongo.ip=127.0.0.1 +mongo.port=13333 +mongo.database.name=test_database +mongo.collection.name=test_collection \ No newline at end of file diff --git a/hadoop-unit-standalone/pom.xml b/hadoop-unit-standalone/pom.xml index 9a323a65..355d5f16 100644 --- a/hadoop-unit-standalone/pom.xml +++ b/hadoop-unit-standalone/pom.xml @@ -11,6 +11,10 @@ hadoop-unit-standalone + + 2.11.3 + + @@ -58,6 +62,12 @@ hadoop-unit-zookeeper + + fr.jetoile.hadoop + hadoop-unit-mongodb + + + commons-configuration commons-configuration @@ -88,6 +98,13 @@ test + + org.mongodb + mongo-java-driver + ${mongo-java-driver.version} + test + + diff --git a/hadoop-unit-standalone/src/main/conf/default.properties b/hadoop-unit-standalone/src/main/conf/default.properties index e2f0aba5..e3a8f58b 100644 --- a/hadoop-unit-standalone/src/main/conf/default.properties +++ b/hadoop-unit-standalone/src/main/conf/default.properties @@ -101,4 +101,10 @@ oozie.purge.local.share.lib.cache=false oozie.sharelib.path=/home/khanh/github oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz oozie.port=20113 -oozie.host=localhost \ No newline at end of file +oozie.host=localhost + +# MongoDB +mongo.ip=127.0.0.1 +mongo.port=13333 +mongo.database.name=test_database +mongo.collection.name=test_collection \ No newline at end of file diff --git a/hadoop-unit-standalone/src/main/conf/hadoop.properties b/hadoop-unit-standalone/src/main/conf/hadoop.properties index a569ccca..96ffca5a 100755 --- a/hadoop-unit-standalone/src/main/conf/hadoop.properties +++ b/hadoop-unit-standalone/src/main/conf/hadoop.properties @@ -7,4 +7,5 @@ hiveserver2=true solrcloud=true #solr=true #oozie=true +#mongodb=true diff --git a/hadoop-unit-standalone/src/main/resources/default.properties b/hadoop-unit-standalone/src/main/resources/default.properties index e2f0aba5..e3a8f58b 100644 --- a/hadoop-unit-standalone/src/main/resources/default.properties +++ b/hadoop-unit-standalone/src/main/resources/default.properties @@ -101,4 +101,10 @@ oozie.purge.local.share.lib.cache=false oozie.sharelib.path=/home/khanh/github oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz oozie.port=20113 -oozie.host=localhost \ No newline at end of file +oozie.host=localhost + +# MongoDB +mongo.ip=127.0.0.1 +mongo.port=13333 +mongo.database.name=test_database +mongo.collection.name=test_collection \ No newline at end of file diff --git a/hadoop-unit-standalone/src/main/resources/hadoop.properties b/hadoop-unit-standalone/src/main/resources/hadoop.properties index a569ccca..96ffca5a 100755 --- a/hadoop-unit-standalone/src/main/resources/hadoop.properties +++ b/hadoop-unit-standalone/src/main/resources/hadoop.properties @@ -7,4 +7,5 @@ hiveserver2=true solrcloud=true #solr=true #oozie=true +#mongodb=true diff --git a/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java b/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java index 2d4f56db..003fe316 100644 --- a/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java +++ b/hadoop-unit-standalone/src/test/java/fr/jetoile/hadoopunit/integrationtest/IntegrationBootstrapTest.java @@ -15,6 +15,8 @@ package fr.jetoile.hadoopunit.integrationtest; +import com.mongodb.*; +import fr.jetoile.hadoopunit.exception.NotFoundServiceException; import fr.jetoile.hadoopunit.test.hdfs.HdfsUtils; import fr.jetoile.hadoopunit.Config; import fr.jetoile.hadoopunit.HadoopBootstrap; @@ -50,13 +52,13 @@ import org.slf4j.LoggerFactory; import java.io.*; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.net.URLConnection; +import java.net.*; import java.sql.*; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.*; +import java.util.Date; import static junit.framework.TestCase.assertNotNull; import static org.fest.assertions.Assertions.assertThat; diff --git a/hadoop-unit-standalone/src/test/resources/default.properties b/hadoop-unit-standalone/src/test/resources/default.properties index 7aeceb48..3036b9b7 100644 --- a/hadoop-unit-standalone/src/test/resources/default.properties +++ b/hadoop-unit-standalone/src/test/resources/default.properties @@ -98,4 +98,15 @@ oozie.purge.local.share.lib.cache=false oozie.sharelib.path=/home/khanh/github oozie.sharelib.name=oozie-4.2.0.2.3.2.0-2950-distro.tar.gz oozie.port=20113 -oozie.host=localhost \ No newline at end of file +oozie.host=localhost + +# MongoDB +mongo.ip=127.0.0.1 +mongo.port=13333 +mongo.database.name=test_database +mongo.collection.name=test_collection + +# Cassandra +cassandra.ip=127.0.0.1 +cassandra.port=13433 +cassandra.temp.dir=/tmp/embedded_cassandra \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5d807c70..f3c304b4 100755 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,7 @@ hadoop-unit-commons hadoop-unit-client hadoop-unit-maven-plugin + hadoop-unit-mongodb @@ -182,6 +183,12 @@ + + com.github.sakserv + hadoop-mini-clusters-mongodb + ${hadoop-mini-clusters.version} + + org.apache.solr solr-solrj