Skip to content

Commit

Permalink
Merge branch 'Arnaud-Nauwynck-master'
Browse files Browse the repository at this point in the history
  • Loading branch information
@jetoile
jetoile committed Oct 18, 2018
2 parents 4b41f55 + 952155e commit b49e372
Show file tree
Hide file tree
Showing 6 changed files with 56 additions and 42 deletions.
7 changes: 7 additions & 0 deletions hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,11 @@ public Configuration loadConfigFile(URL url) throws BootstrapException {
throw new BootstrapException("bad config", e);
}
}

public static String resolveDir(String dir) {
if (dir != null && dir.startsWith("~")) {
dir = System.getProperty("user.home") + dir.substring(1);
}
return dir;
}
}
4 changes: 1 addition & 3 deletions hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/exception/BootstrapException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
*/
package fr.jetoile.hadoopunit.exception;

import org.apache.commons.configuration.ConfigurationException;

public class BootstrapException extends Exception {
public BootstrapException(String s, ConfigurationException e) {
public BootstrapException(String s, Exception e) {
super(s, e);
}

Expand Down
4 changes: 2 additions & 2 deletions hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ private void loadConfig() throws BootstrapException, NotFoundServiceException {
ooziePort = configuration.getInt(HadoopUnitConfig.OOZIE_PORT);
oozieHost = configuration.getString(HadoopUnitConfig.OOZIE_HOST);

oozieShareLibPath = configuration.getString(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY);
oozieShareLibPath = HadoopUtils.resolveDir(configuration.getString(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY));
oozieShareLibName = configuration.getString(HadoopUnitConfig.OOZIE_SHARELIB_NAME_KEY);

List<Object> frameworks = configuration.getList(HadoopUnitConfig.OOZIE_SHARE_LIB_COMPONENT_KEY);
Expand Down Expand Up @@ -266,7 +266,7 @@ public void loadConfig(Map<String, String> configs) {
}

if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY))) {
oozieShareLibPath = configs.get(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY);
oozieShareLibPath = HadoopUtils.resolveDir(configs.get(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY));
}
if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.OOZIE_SHARELIB_NAME_KEY))) {
oozieShareLibName = configs.get(HadoopUnitConfig.OOZIE_SHARELIB_NAME_KEY);
Expand Down
1 change: 1 addition & 0 deletions hadoop-unit-standalone/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/bin/
78 changes: 43 additions & 35 deletions hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public class HadoopStandaloneBootstrap {
static private Configuration hadoopUnitConfiguration;
static private List<Component> componentsToStart = new ArrayList<>();
static private List<ComponentProperties> componentsToStop = new ArrayList<>();
static private List<ComponentProperties> componentsProperty = new ArrayList();
static private List<ComponentProperties> componentsProperty = new ArrayList<>();

static private Settings settings = null;

Expand Down Expand Up @@ -133,27 +133,33 @@ public void serviceCreationFailed(Class<?> type, Class<?> impl, Throwable except
return locator.getService(RepositorySystem.class);
}

public static DefaultRepositorySystemSession newRepositorySystemSession(RepositorySystem system) {
public static DefaultRepositorySystemSession newRepositorySystemSession(RepositorySystem system) throws BootstrapException {
DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
String localRepositoryDir = "";


String mavenHome = getInstalledMavenHome();

if (StringUtils.isNotEmpty(mavenHome)) {
String userHome = System.getProperty("user.home");
if (StringUtils.isNotEmpty(mavenHome)) {
//if MAVEN_HOME or M2_HOME are defined, will read maven's configuration throught settings.xml
LOGGER.info("is going to use the local maven configuration: {}", mavenHome);
Settings settings = getLocalSettings(mavenHome);

localRepositoryDir = settings.getLocalRepository();
if (localRepositoryDir == null) {
LOGGER.debug("is going to use default maven local repository");
localRepositoryDir = System.getProperty("user.home") + "/.m2/repository";
localRepositoryDir = userHome + "/.m2/repository";
}
LOGGER.debug("is going to use {} repository" + localRepositoryDir);
} else {
LOGGER.debug("is going to use the maven repository from {} with key {}", DEFAULT_PROPS_FILE, "maven.local.repo");
localRepositoryDir = hadoopUnitConfiguration.getString("maven.local.repo");
localRepositoryDir = hadoopUnitConfiguration.getString("maven.local.repo");
if (localRepositoryDir != null) {
LOGGER.debug("is going to use the maven repository from {} with key {}", DEFAULT_PROPS_FILE, "maven.local.repo");
localRepositoryDir = HadoopUtils.resolveDir(localRepositoryDir);
} else {
throw new BootstrapException("unable to find M2_HOME/MAVEN_HOME or the configuration key maven.local.repo from "+ DEFAULT_PROPS_FILE);
}
}

LocalRepository localRepo = new LocalRepository(localRepositoryDir);
Expand Down Expand Up @@ -201,17 +207,18 @@ private static Settings getLocalSettings(String mavenHome) {
}

private static String getInstalledMavenHome() {
String maven_home = System.getenv("MAVEN_HOME");
String mavenHome = null;
String m2_home = System.getenv("M2_HOME");

String mavenHome = "";
if (StringUtils.isNotEmpty(maven_home)) {
LOGGER.debug("is going to use MAVEN_HOME to read configuration");
mavenHome = maven_home;
}
if (StringUtils.isNotEmpty(m2_home)) {
LOGGER.debug("is going to use M2_HOME to read configuration");
mavenHome = m2_home;
LOGGER.debug("is going to use M2_HOME to read configuration");
mavenHome = m2_home;
}
if (mavenHome == null) {
String maven_home = System.getenv("MAVEN_HOME"); // legacy, for maven 1
if (StringUtils.isNotEmpty(maven_home)) {
LOGGER.debug("is going to use MAVEN_HOME to read configuration");
mavenHome = maven_home;
}
}
return mavenHome;
}
Expand Down Expand Up @@ -254,29 +261,28 @@ public static void main(String[] args) throws BootstrapException {
}
});

RepositorySystem system = null;
RepositorySystem repositorySystem;
try {
system = getRepositorySystem();
repositorySystem = getRepositorySystem();
} catch (ComponentLookupException e) {
LOGGER.error("unable to get RepositoySystem from external maven", e);
throw new BootstrapException("unable to get RepositoySystem from external maven", e);
}
DefaultRepositorySystemSession session = newRepositorySystemSession(system);
DefaultRepositorySystemSession session = newRepositorySystemSession(repositorySystem);
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.RUNTIME);

RepositorySystem finalSystem = system;
componentsToStart.stream().forEach(c -> {
for(Component c : componentsToStart) {
Artifact artifact = new DefaultArtifact(hadoopUnitConfiguration.getString(c.getArtifactKey()));
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(artifact, JavaScopes.RUNTIME));
collectRequest.setRepositories(getRemoteRepositories());

DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, classpathFilter);

List<ArtifactResult> artifactResults = null;
List<ArtifactResult> artifactResults;
try {
artifactResults = finalSystem.resolveDependencies(session, dependencyRequest).getArtifactResults();
artifactResults = repositorySystem.resolveDependencies(session, dependencyRequest).getArtifactResults();
} catch (DependencyResolutionException e) {
e.printStackTrace();
throw new BootstrapException("failed to resolve dependency artifact " + artifact, e);
}

List<File> artifacts = new ArrayList<>();
Expand All @@ -287,7 +293,7 @@ public static void main(String[] args) throws BootstrapException {

componentsProperty.add(componentProperties);
componentsToStop.add(0, componentProperties);
});
}


Runtime.getRuntime().addShutdownHook(new Thread() {
Expand All @@ -310,10 +316,11 @@ public void run() {
}

private static List<RemoteRepository> getRemoteRepositories() {
if (StringUtils.isEmpty(getInstalledMavenHome())) {
String mavenHome = getInstalledMavenHome();
if (StringUtils.isEmpty(mavenHome)) {
return newRepositories();
} else {
List<Mirror> mirrors = getLocalSettings(getInstalledMavenHome()).getMirrors();
List<Mirror> mirrors = getLocalSettings(mavenHome).getMirrors();
List<RemoteRepository> remoteRepositories = mirrors.stream().map(mirror -> new RemoteRepository.Builder(mirror.getId(), "default", mirror.getUrl()).build()).collect(Collectors.toList());
return remoteRepositories;
}
Expand Down Expand Up @@ -341,8 +348,9 @@ private static void printBanner() {
System.out.println();
}

private static ComponentProperties loadAndRun(String c, String className, List<File> artifacts) {
List<URL> urls = new ArrayList();
@SuppressWarnings("resource")
private static ComponentProperties loadAndRun(String c, String className, List<File> artifacts) {
List<URL> urls = new ArrayList<>();

urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("log4j.xml"));
urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("logback.xml"));
Expand All @@ -364,20 +372,20 @@ private static ComponentProperties loadAndRun(String c, String className, List<F
ClassLoader.getSystemClassLoader().getParent());

// relative to that classloader, find the main class
Class mainClass = null;
Class<?> mainClass;
try {
mainClass = classloader.loadClass(className);
} catch (ClassNotFoundException e) {
LOGGER.error("unable to load class", e);
return null;
}
Method main = null;


try {
Thread.currentThread().setContextClassLoader(classloader);

Object o = mainClass.getConstructor(URL.class).newInstance(HadoopStandaloneBootstrap.class.getClassLoader().getResource(DEFAULT_PROPS_FILE));
main = mainClass.getMethod("start");
Method main = mainClass.getMethod("start");
main.invoke(o);
return new ComponentProperties(o, mainClass);
} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
Expand All @@ -388,9 +396,9 @@ private static ComponentProperties loadAndRun(String c, String className, List<F

private static class ComponentProperties {
private Object instance;
private Class mainClass;
private Class<?> mainClass;

public ComponentProperties(Object instance, Class mainClass) {
public ComponentProperties(Object instance, Class<?> mainClass) {
this.instance = instance;
this.mainClass = mainClass;
}
Expand All @@ -399,7 +407,7 @@ public Object getInstance() {
return instance;
}

public Class getMainClass() {
public Class<?> getMainClass() {
return mainClass;
}
}
Expand Down
4 changes: 2 additions & 2 deletions hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ confluent.ksql_rest.artifact=fr.jetoile.hadoop:hadoop-unit-confluent:2.10-SNAPSH
confluent.kafka_rest.artifact=fr.jetoile.hadoop:hadoop-unit-confluent-rest:2.10-SNAPSHOT

maven.central.repo=https://repo.maven.apache.org/maven2/
maven.local.repo=/home/khanh/.m2/repository
maven.local.repo=~/.m2/repository

maven.debug=true

Expand Down Expand Up @@ -139,7 +139,7 @@ oozie.hdfs.share.lib.dir=/tmp/share_lib
oozie.share.lib.create=true
oozie.local.share.lib.cache.dir=/tmp/share_lib_cache
oozie.purge.local.share.lib.cache=false
oozie.sharelib.path=/home/khanh/github
oozie.sharelib.path=~/github
oozie.sharelib.name=oozie-4.2.0.2.6.3.0-235-distro.tar.gz
oozie.port=20113
oozie.host=localhost
Expand Down

0 comments on commit b49e372

Please sign in to comment.