From 2d324c3634825e1bd19946a0e180ccd2cf859d2a Mon Sep 17 00:00:00 2001 From: jetoile Date: Tue, 29 Oct 2019 17:13:23 +0100 Subject: [PATCH] add bookkeeper --- hadoop-unit-bookkeeper/pom.xml | 69 ++++++ .../component/BookkeeperBootstrap.java | 196 ++++++++++++++++++ .../component/BookkeeperConfig.java | 25 +++ .../component/BookkeeperMetadata.java | 34 +++ .../fr.jetoile.hadoopunit.component.Bootstrap | 1 + .../resources/hadoop-unit-default.properties | 14 ++ .../src/main/resources/logback.xml | 29 +++ .../component/BookkeeperBootstrapTest.java | 73 +++++++ .../src/site/markdown/what-is-hadoop-unit.md | 1 + .../resources/hadoop-unit-default.properties | 12 +- .../src/main/resources/hadoop.properties | 1 + pom.xml | 3 +- 12 files changed, 456 insertions(+), 2 deletions(-) create mode 100644 hadoop-unit-bookkeeper/pom.xml create mode 100644 hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrap.java create mode 100644 hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperConfig.java create mode 100644 hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperMetadata.java create mode 100644 hadoop-unit-bookkeeper/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap create mode 100644 hadoop-unit-bookkeeper/src/main/resources/hadoop-unit-default.properties create mode 100755 hadoop-unit-bookkeeper/src/main/resources/logback.xml create mode 100644 hadoop-unit-bookkeeper/src/test/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrapTest.java diff --git a/hadoop-unit-bookkeeper/pom.xml b/hadoop-unit-bookkeeper/pom.xml new file mode 100644 index 00000000..d72b1cf1 --- /dev/null +++ b/hadoop-unit-bookkeeper/pom.xml @@ -0,0 +1,69 @@ + + + + + hadoop-unit + fr.jetoile.hadoop + 3.6-SNAPSHOT + + 4.0.0 + + hadoop-unit-bookkeeper + + + 4.9.2 + 4.1.32.Final + + + + + fr.jetoile.hadoop + hadoop-unit-commons + + + + fr.jetoile.hadoop + hadoop-unit-zookeeper + + + + io.netty + netty-all + ${netty.version} + + + + org.apache.bookkeeper + bookkeeper-server + ${bookkeeper.version} + + + + org.apache.bookkeeper.http + vertx-http-server + ${bookkeeper.version} + + + + ch.qos.logback + logback-classic + + + + javax.ws.rs + javax.ws.rs-api + 2.1.1 + test + + + + org.jboss.resteasy + resteasy-client + 4.3.1.Final + test + + + + \ No newline at end of file diff --git a/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrap.java b/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrap.java new file mode 100644 index 00000000..92de6474 --- /dev/null +++ b/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrap.java @@ -0,0 +1,196 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fr.jetoile.hadoopunit.component; + +import fr.jetoile.hadoopunit.ComponentMetadata; +import fr.jetoile.hadoopunit.HadoopUtils; +import fr.jetoile.hadoopunit.exception.BootstrapException; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.common.component.ComponentStarter; +import org.apache.bookkeeper.common.component.LifecycleComponent; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.server.conf.BookieConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.Map; + +import static org.apache.bookkeeper.server.Main.buildBookieServer; + +public class BookkeeperBootstrap implements Bootstrap { + static final private Logger LOGGER = LoggerFactory.getLogger(BookkeeperBootstrap.class); + private State state = State.STOPPED; + private Configuration configuration; + + private LifecycleComponent server; + private ServerConfiguration bookkeeperConf; + + private int port; + private String ip; + private String tmpDirPath; + private int httpPort; + + private int zookeeperPort; + private String zookeeperHost; + + public BookkeeperBootstrap() { + try { + configuration = HadoopUtils.INSTANCE.loadConfigFile(null); + loadConfig(); + } catch (BootstrapException e) { + LOGGER.error("unable to load configuration", e); + } + } + + public BookkeeperBootstrap(URL url) { + try { + configuration = HadoopUtils.INSTANCE.loadConfigFile(url); + loadConfig(); + } catch (BootstrapException e) { + LOGGER.error("unable to load configuration", e); + } + } + + @Override + public ComponentMetadata getMetadata() { + return new BookkeeperMetadata(); + } + + @Override + public String getProperties() { + return "\n \t\t\t ip:" + ip + + "\n \t\t\t port:" + port + + "\n \t\t\t http port:" + httpPort; + } + + private void loadConfig() { + port = configuration.getInt(BookkeeperConfig.BOOKKEEPER_PORT_KEY); + ip = configuration.getString(BookkeeperConfig.BOOKKEEPER_IP_KEY); + httpPort = configuration.getInt(BookkeeperConfig.BOOKKEEPER_HTTP_PORT_KEY); + tmpDirPath = getTmpDirPath(configuration, BookkeeperConfig.BOOKKEEPER_TEMP_DIR_KEY); + + zookeeperPort = configuration.getInt(ZookeeperConfig.ZOOKEEPER_PORT_KEY); + zookeeperHost = configuration.getString(ZookeeperConfig.ZOOKEEPER_HOST_KEY); + } + + @Override + public void loadConfig(Map configs) { + if (StringUtils.isNotEmpty(configs.get(BookkeeperConfig.BOOKKEEPER_PORT_KEY))) { + port = Integer.parseInt(configs.get(BookkeeperConfig.BOOKKEEPER_PORT_KEY)); + } + if (StringUtils.isNotEmpty(configs.get(BookkeeperConfig.BOOKKEEPER_IP_KEY))) { + ip = configs.get(BookkeeperConfig.BOOKKEEPER_IP_KEY); + } + if (StringUtils.isNotEmpty(configs.get(BookkeeperConfig.BOOKKEEPER_TEMP_DIR_KEY))) { + tmpDirPath = getTmpDirPath(configs, BookkeeperConfig.BOOKKEEPER_TEMP_DIR_KEY); + } + } + + private void build() throws Exception { + bookkeeperConf = new ServerConfiguration(); + bookkeeperConf.setAdvertisedAddress(ip); + bookkeeperConf.setBookiePort(port); + bookkeeperConf.setHttpServerEnabled(true); + bookkeeperConf.setHttpServerPort(httpPort); + bookkeeperConf.setMetadataServiceUri("zk+hierarchical://" + zookeeperHost + ":" + zookeeperPort + "/ledgers"); + bookkeeperConf.setJournalDirName(tmpDirPath + "/bk-txn"); + bookkeeperConf.setLedgerDirNames(new String[] {tmpDirPath + "/ledger"}); + bookkeeperConf.setIndexDirName(new String[] {tmpDirPath + "/index"}); + + bookkeeperConf.addProperty("httpServerClass", "org.apache.bookkeeper.http.vertx.VertxHttpServer"); + + bookkeeperConf.setWriteBufferBytes(65536); + + bookkeeperConf.setZkEnableSecurity(false); + bookkeeperConf.setZkTimeout(10000); + + BookKeeperAdmin.format(bookkeeperConf, false, true); + + server = buildBookieServer(new BookieConfiguration(bookkeeperConf)); + + } + + @Override + public Bootstrap start() { + if (state == State.STOPPED) { + state = State.STARTING; + LOGGER.info("{} is starting", this.getClass().getName()); + try { + build(); + + StartBookkeeper startBookkeeper = new StartBookkeeper(server); + Thread t = new Thread(startBookkeeper); + t.setDaemon(true); + t.start(); + } catch (Exception e) { + LOGGER.error("unable to add bookkeeper", e); + } + state = State.STARTED; + LOGGER.info("{} is started", this.getClass().getName()); + } + + return this; + } + + @Override + public Bootstrap stop() { + if (state == State.STARTED) { + state = State.STOPPING; + LOGGER.info("{} is stopping", this.getClass().getName()); + try { + cleanup(); + } catch (Exception e) { + LOGGER.error("unable to stop bookkeeper", e); + } + state = State.STOPPED; + LOGGER.info("{} is stopped", this.getClass().getName()); + } + return this; + } + + private void cleanup() { + try { + FileUtils.deleteDirectory(Paths.get(tmpDirPath).toFile()); + } catch (IOException e) { + LOGGER.error("unable to delete {}", tmpDirPath, e); + } + } + + private static class StartBookkeeper implements Runnable { + + private LifecycleComponent server; + + public StartBookkeeper(LifecycleComponent server) { + this.server = server; + } + + @Override + public void run() { + try { + ComponentStarter.startComponent(server).get(); + } catch (Throwable t) { + t.printStackTrace(); + } + } + } +} + + diff --git a/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperConfig.java b/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperConfig.java new file mode 100644 index 00000000..42685602 --- /dev/null +++ b/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperConfig.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fr.jetoile.hadoopunit.component; + +public class BookkeeperConfig { + + // Cassandra + public static final String BOOKKEEPER_IP_KEY = "bookkeeper.ip"; + public static final String BOOKKEEPER_PORT_KEY = "bookkeeper.port"; + public static final String BOOKKEEPER_HTTP_PORT_KEY = "bookkeeper.http.port"; + public static final String BOOKKEEPER_TEMP_DIR_KEY = "bookkeeper.temp.dir"; + + private BookkeeperConfig() {} +} diff --git a/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperMetadata.java b/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperMetadata.java new file mode 100644 index 00000000..885ed8aa --- /dev/null +++ b/hadoop-unit-bookkeeper/src/main/java/fr/jetoile/hadoopunit/component/BookkeeperMetadata.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fr.jetoile.hadoopunit.component; + +import fr.jetoile.hadoopunit.ComponentMetadata; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class BookkeeperMetadata extends ComponentMetadata { + + @Override + public String getName() { + return "BOOKKEEPER"; + } + + @Override + public List getDependencies() { + return Arrays.asList("ZOOKEEPER"); + } + +} diff --git a/hadoop-unit-bookkeeper/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap b/hadoop-unit-bookkeeper/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap new file mode 100644 index 00000000..f644647f --- /dev/null +++ b/hadoop-unit-bookkeeper/src/main/resources/META-INF/services/fr.jetoile.hadoopunit.component.Bootstrap @@ -0,0 +1 @@ +fr.jetoile.hadoopunit.component.BookkeeperBootstrap \ No newline at end of file diff --git a/hadoop-unit-bookkeeper/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-bookkeeper/src/main/resources/hadoop-unit-default.properties new file mode 100644 index 00000000..913adf6f --- /dev/null +++ b/hadoop-unit-bookkeeper/src/main/resources/hadoop-unit-default.properties @@ -0,0 +1,14 @@ +#HADOOP_HOME=/opt/hadoop + +tmp.dir.path=/tmp + +# Zookeeper +zookeeper.temp.dir=/embedded_zk +zookeeper.host=127.0.0.1 +zookeeper.port=22010 + +# BookKeeper +bookkeeper.ip=127.0.0.1 +bookkeeper.port=31810 +bookkeeper.http.port=31900 +bookkeeper.temp.dir=/bookeeper \ No newline at end of file diff --git a/hadoop-unit-bookkeeper/src/main/resources/logback.xml b/hadoop-unit-bookkeeper/src/main/resources/logback.xml new file mode 100755 index 00000000..ceb6d6ca --- /dev/null +++ b/hadoop-unit-bookkeeper/src/main/resources/logback.xml @@ -0,0 +1,29 @@ + + + + %d{HH:mm:ss.SSS} %-5level %logger{36} - %msg %n + + + + + + + true + + %d{HH:mm:ss.SSS} %-5level %logger{36} - %green(%msg) %n + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-unit-bookkeeper/src/test/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrapTest.java b/hadoop-unit-bookkeeper/src/test/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrapTest.java new file mode 100644 index 00000000..df9d1d9c --- /dev/null +++ b/hadoop-unit-bookkeeper/src/test/java/fr/jetoile/hadoopunit/component/BookkeeperBootstrapTest.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fr.jetoile.hadoopunit.component; + +import fr.jetoile.hadoopunit.HadoopBootstrap; +import fr.jetoile.hadoopunit.HadoopUnitConfig; +import fr.jetoile.hadoopunit.exception.BootstrapException; +import fr.jetoile.hadoopunit.exception.NotFoundServiceException; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.core.Response; + +import static org.assertj.core.api.Assertions.assertThat; + +public class BookkeeperBootstrapTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BookkeeperBootstrapTest.class); + + static private Configuration configuration; + + + @BeforeClass + public static void setup() throws BootstrapException { + HadoopBootstrap.INSTANCE.startAll(); + + try { + configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE); + } catch (ConfigurationException e) { + throw new BootstrapException("bad config", e); + } + } + + + @AfterClass + public static void tearDown() throws BootstrapException { + HadoopBootstrap.INSTANCE.stopAll(); + } + + @Test + public void bookkeeperShouldStart() throws NotFoundServiceException { + + Client client = ClientBuilder.newClient(); + String uri = "http://localhost:" + configuration.getInt(BookkeeperConfig.BOOKKEEPER_HTTP_PORT_KEY); + + Response hearbeatResponse = client.target(uri + "/heartbeat").request().get(); + assertThat(hearbeatResponse.getStatus()).isEqualTo(200); + + Response bookieStateResponse = client.target(uri + "/api/v1/bookie/state").request().get(); + assertThat(hearbeatResponse.getStatus()).isEqualTo(200); + + } +} \ No newline at end of file diff --git a/hadoop-unit-site/src/site/markdown/what-is-hadoop-unit.md b/hadoop-unit-site/src/site/markdown/what-is-hadoop-unit.md index f997e119..4b4a7b58 100644 --- a/hadoop-unit-site/src/site/markdown/what-is-hadoop-unit.md +++ b/hadoop-unit-site/src/site/markdown/what-is-hadoop-unit.md @@ -80,5 +80,6 @@ The available components are: * KNOX * ALLUXIO * PULSAR +* BOOKKEEPER * REDIS (for linux and macOS only) diff --git a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties index 98e65115..cf3e137a 100644 --- a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties @@ -114,6 +114,10 @@ pulsar_standalone.artifactId=fr.jetoile.hadoop:hadoop-unit-pulsar-standalone:${p pulsar_standalone.mainClass=fr.jetoile.hadoopunit.component.PulsarStandaloneBootstrap pulsar_standalone.metadataClass=fr.jetoile.hadoopunit.component.PulsarStandaloneMetadata +bookkeeper.artifactId=fr.jetoile.hadoop:hadoop-unit-bookkeeper:${project.version} +bookkeeper.mainClass=fr.jetoile.hadoopunit.component.BookkeeperBootstrap +bookkeeper.metadataClass=fr.jetoile.hadoopunit.component.BookkeeperMetadata + maven.central.repo=https://repo.maven.apache.org/maven2/ maven.local.repo=~/.m2/repository @@ -390,4 +394,10 @@ pulsar.zookeeper.temp.dir=/pulsar/embedded_zk pulsar.zookeeper.port=22020 pulsar.ip=127.0.0.1 pulsar.port=22022 -pulsar.temp.dir=/pulsar \ No newline at end of file +pulsar.temp.dir=/pulsar + +# BookKeeper +bookkeeper.ip=127.0.0.1 +bookkeeper.port=31810 +bookkeeper.http.port=31900 +bookkeeper.temp.dir=/bookeeper \ No newline at end of file diff --git a/hadoop-unit-standalone/src/main/resources/hadoop.properties b/hadoop-unit-standalone/src/main/resources/hadoop.properties index dd1d3d5b..91b4e7c3 100755 --- a/hadoop-unit-standalone/src/main/resources/hadoop.properties +++ b/hadoop-unit-standalone/src/main/resources/hadoop.properties @@ -25,3 +25,4 @@ hiveserver23=true #docker=true #docker_compose=true pulsar_standalone=true +#bookkeeper=true diff --git a/pom.xml b/pom.xml index 07d839cc..6b651103 100755 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ hadoop-unit-hiveserver23 hadoop-unit-hivemeta3 hadoop-unit-zookeeper + hadoop-unit-bookkeeper hadoop-unit-solr hadoop-unit-solrcloud hadoop-unit-hdfs @@ -65,7 +66,7 @@ hadoop-unit-commons-hadoop3 hadoop-unit-client hadoop-unit-maven-plugin - hadoop-unit-cassandra + hadoop-unit-cassandra hadoop-unit-pulsar-standalone hadoop-unit-mongodb hadoop-unit-elasticsearch