Skip to content

Commit

Permalink
Make commons' modules independant from plugins (#89)
Browse files Browse the repository at this point in the history
  • Loading branch information
jetoile authored Jan 28, 2019
1 parent b4b18f0 commit f7e7184
Show file tree
Hide file tree
Showing 159 changed files with 3,525 additions and 1,445 deletions.
2 changes: 1 addition & 1 deletion hadoop-unit-alluxio/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>hadoop-unit</artifactId>
<groupId>fr.jetoile.hadoop</groupId>
<version>2.11-SNAPSHOT</version>
<version>3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,21 @@

import alluxio.PropertyKey;
import alluxio.master.LocalAlluxioCluster;
import alluxio.multi.process.MultiProcessCluster;
import alluxio.security.LoginUserTestUtils;
import alluxio.security.authentication.AuthenticatedClientUser;
import fr.jetoile.hadoopunit.Component;
import fr.jetoile.hadoopunit.HadoopUnitConfig;
import fr.jetoile.hadoopunit.ComponentMetadata;
import fr.jetoile.hadoopunit.HadoopUtils;
import fr.jetoile.hadoopunit.exception.BootstrapException;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

public class AlluxioBootstrap implements Bootstrap {
final public static String NAME = Component.ALLUXIO.name();

static final private Logger LOGGER = LoggerFactory.getLogger(AlluxioBootstrap.class);

private LocalAlluxioCluster alluxioLocalCluster;
Expand Down Expand Up @@ -77,8 +68,8 @@ public AlluxioBootstrap(URL url) {
}

@Override
public String getName() {
return NAME;
public ComponentMetadata getMetadata() {
return new AlluxioMetadata();
}

@Override
Expand Down Expand Up @@ -126,45 +117,45 @@ private void build() {
}

private void loadConfig() throws BootstrapException {
this.workDirectory = configuration.getString(HadoopUnitConfig.ALLUXIO_WORK_DIR);
this.hostname = configuration.getString(HadoopUnitConfig.ALLUXIO_HOSTNAME);
this.masterRpcPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_MASTER_RPC_PORT);
this.masterWebPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_MASTER_WEB_PORT);
this.proxyWebPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_PROXY_WEB_PORT);
this.workerRpcPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_WORKER_RPC_PORT);
this.workerDataPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_WORKER_DATA_PORT);
this.workerWebPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_WORKER_WEB_PORT);
this.webappDirectory = configuration.getString(HadoopUnitConfig.ALLUXIO_WEBAPP_DIRECTORY);
this.workDirectory = configuration.getString(AlluxioConfig.ALLUXIO_WORK_DIR);
this.hostname = configuration.getString(AlluxioConfig.ALLUXIO_HOSTNAME);
this.masterRpcPort = configuration.getInt(AlluxioConfig.ALLUXIO_MASTER_RPC_PORT);
this.masterWebPort = configuration.getInt(AlluxioConfig.ALLUXIO_MASTER_WEB_PORT);
this.proxyWebPort = configuration.getInt(AlluxioConfig.ALLUXIO_PROXY_WEB_PORT);
this.workerRpcPort = configuration.getInt(AlluxioConfig.ALLUXIO_WORKER_RPC_PORT);
this.workerDataPort = configuration.getInt(AlluxioConfig.ALLUXIO_WORKER_DATA_PORT);
this.workerWebPort = configuration.getInt(AlluxioConfig.ALLUXIO_WORKER_WEB_PORT);
this.webappDirectory = configuration.getString(AlluxioConfig.ALLUXIO_WEBAPP_DIRECTORY);
}

@Override
public void loadConfig(Map<String, String> configs) {
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_WORK_DIR))) {
this.workDirectory = configuration.getString(HadoopUnitConfig.ALLUXIO_WORK_DIR);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_WORK_DIR))) {
this.workDirectory = configuration.getString(AlluxioConfig.ALLUXIO_WORK_DIR);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_HOSTNAME))) {
this.hostname = configuration.getString(HadoopUnitConfig.ALLUXIO_HOSTNAME);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_HOSTNAME))) {
this.hostname = configuration.getString(AlluxioConfig.ALLUXIO_HOSTNAME);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_MASTER_RPC_PORT))) {
this.masterRpcPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_MASTER_RPC_PORT);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_MASTER_RPC_PORT))) {
this.masterRpcPort = configuration.getInt(AlluxioConfig.ALLUXIO_MASTER_RPC_PORT);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_MASTER_WEB_PORT))) {
this.masterWebPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_MASTER_WEB_PORT);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_MASTER_WEB_PORT))) {
this.masterWebPort = configuration.getInt(AlluxioConfig.ALLUXIO_MASTER_WEB_PORT);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_PROXY_WEB_PORT))) {
this.proxyWebPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_PROXY_WEB_PORT);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_PROXY_WEB_PORT))) {
this.proxyWebPort = configuration.getInt(AlluxioConfig.ALLUXIO_PROXY_WEB_PORT);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_WORKER_RPC_PORT))) {
this.workerRpcPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_WORKER_RPC_PORT);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_WORKER_RPC_PORT))) {
this.workerRpcPort = configuration.getInt(AlluxioConfig.ALLUXIO_WORKER_RPC_PORT);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_WORKER_DATA_PORT))) {
this.workerDataPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_WORKER_DATA_PORT);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_WORKER_DATA_PORT))) {
this.workerDataPort = configuration.getInt(AlluxioConfig.ALLUXIO_WORKER_DATA_PORT);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_WORKER_WEB_PORT))) {
this.workerWebPort = configuration.getInt(HadoopUnitConfig.ALLUXIO_WORKER_WEB_PORT);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_WORKER_WEB_PORT))) {
this.workerWebPort = configuration.getInt(AlluxioConfig.ALLUXIO_WORKER_WEB_PORT);
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.ALLUXIO_HOSTNAME))) {
this.webappDirectory = configuration.getString(HadoopUnitConfig.ALLUXIO_WEBAPP_DIRECTORY);
if (StringUtils.isNotEmpty(configs.get(AlluxioConfig.ALLUXIO_HOSTNAME))) {
this.webappDirectory = configuration.getString(AlluxioConfig.ALLUXIO_WEBAPP_DIRECTORY);
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package fr.jetoile.hadoopunit.component;

public class AlluxioConfig {

// Alluxio
public static final String ALLUXIO_WORK_DIR = "alluxio.work.dir";
public static final String ALLUXIO_HOSTNAME = "alluxio.hostname";
public static final String ALLUXIO_MASTER_RPC_PORT = "alluxio.master.port";
public static final String ALLUXIO_MASTER_WEB_PORT = "alluxio.master.web.port";
public static final String ALLUXIO_PROXY_WEB_PORT = "alluxio.proxy.web.port";
public static final String ALLUXIO_WORKER_RPC_PORT = "alluxio.worker.port";
public static final String ALLUXIO_WORKER_DATA_PORT = "alluxio.worker.data.port";
public static final String ALLUXIO_WORKER_WEB_PORT = "alluxio.worker.web.port";
public static final String ALLUXIO_WEBAPP_DIRECTORY = "alluxio.webapp.directory";

private AlluxioConfig() {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package fr.jetoile.hadoopunit.component;

import fr.jetoile.hadoopunit.ComponentMetadata;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class AlluxioMetadata extends ComponentMetadata {

@Override
public String getName() {
return "ALLUXIO";
}

@Override
public String getBootstrapClass() {
return "fr.jetoile.hadoopunit.component.AlluxioBootstrap";
}

@Override
public String getArtifactKey() {
return "alluxio.artifact";
}

@Override
public List<String> getDependencies() {
return Arrays.asList("HDFS");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ private boolean readFile(FileSystem fs) throws IOException, AlluxioException {
private FileSystemMasterClient buildFsMasterClient() {
MasterClientConfig masterClientConfig = new MasterClientConfig()
.withMasterInquireClient(new SingleMasterInquireClient(
new InetSocketAddress(configuration.getString(HadoopUnitConfig.ALLUXIO_HOSTNAME), configuration.getInt(HadoopUnitConfig.ALLUXIO_MASTER_RPC_PORT)))
new InetSocketAddress(configuration.getString(AlluxioConfig.ALLUXIO_HOSTNAME), configuration.getInt(AlluxioConfig.ALLUXIO_MASTER_RPC_PORT)))
);

FileSystemMasterClient fsMasterClient = FileSystemMasterClient.Factory.create(masterClientConfig);
Expand Down
2 changes: 1 addition & 1 deletion hadoop-unit-cassandra/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<parent>
<artifactId>hadoop-unit</artifactId>
<groupId>fr.jetoile.hadoop</groupId>
<version>2.11-SNAPSHOT</version>
<version>3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,14 @@
package fr.jetoile.hadoopunit.component;

import com.datastax.driver.core.Session;
import fr.jetoile.hadoopunit.Component;
import fr.jetoile.hadoopunit.HadoopUnitConfig;
import fr.jetoile.hadoopunit.ComponentMetadata;
import fr.jetoile.hadoopunit.HadoopUtils;
import fr.jetoile.hadoopunit.exception.BootstrapException;
import info.archinnov.achilles.embedded.CassandraEmbeddedServerBuilder;
import info.archinnov.achilles.embedded.CassandraShutDownHook;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -37,8 +33,6 @@
import java.util.Map;

public class CassandraBootstrap implements Bootstrap {
final public static String NAME = Component.CASSANDRA.name();

static final private Logger LOGGER = LoggerFactory.getLogger(CassandraBootstrap.class);
public Session session;
private State state = State.STOPPED;
Expand Down Expand Up @@ -66,8 +60,8 @@ public CassandraBootstrap(URL url) {
}

@Override
public String getName() {
return NAME;
public ComponentMetadata getMetadata() {
return new CassandraMetadata();
}

@Override
Expand All @@ -76,27 +70,27 @@ public String getProperties() {
"\n \t\t\t port:" + port;
}

private void loadConfig() throws BootstrapException {
port = configuration.getInt(HadoopUnitConfig.CASSANDRA_PORT_KEY);
ip = configuration.getString(HadoopUnitConfig.CASSANDRA_IP_KEY);
private void loadConfig() {
port = configuration.getInt(CassandraConfig.CASSANDRA_PORT_KEY);
ip = configuration.getString(CassandraConfig.CASSANDRA_IP_KEY);
}

@Override
public void loadConfig(Map<String, String> configs) {
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.CASSANDRA_PORT_KEY))) {
port = Integer.parseInt(configs.get(HadoopUnitConfig.CASSANDRA_PORT_KEY));
if (StringUtils.isNotEmpty(configs.get(CassandraConfig.CASSANDRA_PORT_KEY))) {
port = Integer.parseInt(configs.get(CassandraConfig.CASSANDRA_PORT_KEY));
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.CASSANDRA_IP_KEY))) {
ip = configs.get(HadoopUnitConfig.CASSANDRA_IP_KEY);
if (StringUtils.isNotEmpty(configs.get(CassandraConfig.CASSANDRA_IP_KEY))) {
ip = configs.get(CassandraConfig.CASSANDRA_IP_KEY);
}
}

private void build() throws InterruptedException, IOException, TTransportException {
Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY)));
Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/data"));
Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog"));
Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches"));
Files.createDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints"));
private void build() throws IOException {
Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY)));
Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/data"));
Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog"));
Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches"));
Files.createDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints"));

shutdownHook = new CassandraShutDownHook();

Expand All @@ -106,10 +100,10 @@ private void build() throws InterruptedException, IOException, TTransportExcepti
.withBroadcastAddress(ip)
.withBroadcastRpcAddress(ip)
.withCQLPort(port)
.withDataFolder(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/data")
.withCommitLogFolder(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog")
.withSavedCachesFolder(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches")
.withHintsFolder(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints")
.withDataFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/data")
.withCommitLogFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/commitlog")
.withSavedCachesFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/saved_caches")
.withHintsFolder(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY) + "/hints")
.cleanDataFilesAtStartup(true)
.withShutdownHook(shutdownHook)
.cleanDataFilesAtStartup(true)
Expand Down Expand Up @@ -153,9 +147,9 @@ public Bootstrap stop() {

private void cleanup() {
try {
FileUtils.deleteDirectory(Paths.get(configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY)).toFile());
FileUtils.deleteDirectory(Paths.get(configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY)).toFile());
} catch (IOException e) {
LOGGER.error("unable to delete {}", configuration.getString(HadoopUnitConfig.CASSANDRA_TEMP_DIR_KEY), e);
LOGGER.error("unable to delete {}", configuration.getString(CassandraConfig.CASSANDRA_TEMP_DIR_KEY), e);
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package fr.jetoile.hadoopunit.component;

public class CassandraConfig {

// Cassandra
public static final String CASSANDRA_IP_KEY = "cassandra.ip";
public static final String CASSANDRA_PORT_KEY = "cassandra.port";
public static final String CASSANDRA_TEMP_DIR_KEY = "cassandra.temp.dir";

private CassandraConfig() {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package fr.jetoile.hadoopunit.component;

import fr.jetoile.hadoopunit.ComponentMetadata;

import java.util.Collections;
import java.util.List;

public class CassandraMetadata extends ComponentMetadata {

@Override
public String getName() {
return "CASSANDRA";
}

@Override
public String getBootstrapClass() {
return "fr.jetoile.hadoopunit.component.CassandraBootstrap";
}

@Override
public String getArtifactKey() {
return "cassandra.artifact";
}

@Override
public List<String> getDependencies() {
return Collections.EMPTY_LIST;
}

}
Loading

0 comments on commit f7e7184

Please sign in to comment.