From 67e4eb8d5632a3560cd9314e526f9047692a7b67 Mon Sep 17 00:00:00 2001 From: Arnaud Nauwynck Date: Thu, 18 Oct 2018 08:26:35 +0200 Subject: [PATCH 1/3] fix maven_home detection + /home/khan hard-coded in default configuration --- .../fr/jetoile/hadoopunit/HadoopUtils.java | 7 ++ .../exception/BootstrapException.java | 4 +- .../hadoopunit/component/OozieBootstrap.java | 4 +- hadoop-unit-standalone/.gitignore | 1 + .../hadoopunit/HadoopStandaloneBootstrap.java | 92 ++++++++++++------- .../resources/hadoop-unit-default.properties | 4 +- hadoop-unit-standalone/test-console.sh | 12 +++ 7 files changed, 83 insertions(+), 41 deletions(-) create mode 100644 hadoop-unit-standalone/.gitignore create mode 100755 hadoop-unit-standalone/test-console.sh diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUtils.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUtils.java index b9d5b60a..f53a769d 100644 --- a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUtils.java +++ b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUtils.java @@ -148,4 +148,11 @@ public Configuration loadConfigFile(URL url) throws BootstrapException { throw new BootstrapException("bad config", e); } } + + public static String resolveDir(String dir) { + if (dir != null && dir.startsWith("~")) { + dir = System.getProperty("user.home") + dir.substring(1); + } + return dir; + } } diff --git a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/exception/BootstrapException.java b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/exception/BootstrapException.java index c34f3ff6..647b8f9b 100644 --- a/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/exception/BootstrapException.java +++ b/hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/exception/BootstrapException.java @@ -13,10 +13,8 @@ */ package fr.jetoile.hadoopunit.exception; -import org.apache.commons.configuration.ConfigurationException; - public class BootstrapException extends Exception { - public BootstrapException(String s, ConfigurationException e) { + public BootstrapException(String s, Exception e) { super(s, e); } diff --git a/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java b/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java index 0d045457..b0d5d72c 100644 --- a/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java +++ b/hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java @@ -195,7 +195,7 @@ private void loadConfig() throws BootstrapException, NotFoundServiceException { ooziePort = configuration.getInt(HadoopUnitConfig.OOZIE_PORT); oozieHost = configuration.getString(HadoopUnitConfig.OOZIE_HOST); - oozieShareLibPath = configuration.getString(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY); + oozieShareLibPath = HadoopUtils.resolveDir(configuration.getString(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY)); oozieShareLibName = configuration.getString(HadoopUnitConfig.OOZIE_SHARELIB_NAME_KEY); List frameworks = configuration.getList(HadoopUnitConfig.OOZIE_SHARE_LIB_COMPONENT_KEY); @@ -266,7 +266,7 @@ public void loadConfig(Map configs) { } if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY))) { - oozieShareLibPath = configs.get(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY); + oozieShareLibPath = HadoopUtils.resolveDir(configs.get(HadoopUnitConfig.OOZIE_SHARELIB_PATH_KEY)); } if (StringUtils.isNotEmpty(configs.get(HadoopUnitConfig.OOZIE_SHARELIB_NAME_KEY))) { oozieShareLibName = configs.get(HadoopUnitConfig.OOZIE_SHARELIB_NAME_KEY); diff --git a/hadoop-unit-standalone/.gitignore b/hadoop-unit-standalone/.gitignore new file mode 100644 index 00000000..ae3c1726 --- /dev/null +++ b/hadoop-unit-standalone/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java b/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java index b852f489..b70d59e4 100644 --- a/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java +++ b/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java @@ -75,7 +75,7 @@ public class HadoopStandaloneBootstrap { static private Configuration hadoopUnitConfiguration; static private List componentsToStart = new ArrayList<>(); static private List componentsToStop = new ArrayList<>(); - static private List componentsProperty = new ArrayList(); + static private List componentsProperty = new ArrayList<>(); static private Settings settings = null; @@ -140,7 +140,8 @@ public static DefaultRepositorySystemSession newRepositorySystemSession(Reposito String mavenHome = getInstalledMavenHome(); - if (StringUtils.isNotEmpty(mavenHome)) { + String userHome = System.getProperty("user.home"); + if (StringUtils.isNotEmpty(mavenHome)) { //if MAVEN_HOME or M2_HOME are defined, will read maven's configuration throught settings.xml LOGGER.info("is going to use the local maven configuration: {}", mavenHome); Settings settings = getLocalSettings(mavenHome); @@ -148,12 +149,18 @@ public static DefaultRepositorySystemSession newRepositorySystemSession(Reposito localRepositoryDir = settings.getLocalRepository(); if (localRepositoryDir == null) { LOGGER.debug("is going to use default maven local repository"); - localRepositoryDir = System.getProperty("user.home") + "/.m2/repository"; + localRepositoryDir = userHome + "/.m2/repository"; } LOGGER.debug("is going to use {} repository" + localRepositoryDir); } else { - LOGGER.debug("is going to use the maven repository from {} with key {}", DEFAULT_PROPS_FILE, "maven.local.repo"); - localRepositoryDir = hadoopUnitConfiguration.getString("maven.local.repo"); + localRepositoryDir = hadoopUnitConfiguration.getString("maven.local.repo"); + if (localRepositoryDir != null) { + LOGGER.debug("is going to use the maven repository from {} with key {}", DEFAULT_PROPS_FILE, "maven.local.repo"); + localRepositoryDir = HadoopUtils.resolveDir(localRepositoryDir); + } else { + // using default + localRepositoryDir = userHome + "/.m2/repository"; + } } LocalRepository localRepo = new LocalRepository(localRepositoryDir); @@ -201,17 +208,32 @@ private static Settings getLocalSettings(String mavenHome) { } private static String getInstalledMavenHome() { - String maven_home = System.getenv("MAVEN_HOME"); + String mavenHome = null; String m2_home = System.getenv("M2_HOME"); - - String mavenHome = ""; - if (StringUtils.isNotEmpty(maven_home)) { - LOGGER.debug("is going to use MAVEN_HOME to read configuration"); - mavenHome = maven_home; - } if (StringUtils.isNotEmpty(m2_home)) { - LOGGER.debug("is going to use M2_HOME to read configuration"); - mavenHome = m2_home; + LOGGER.debug("is going to use M2_HOME to read configuration"); + mavenHome = m2_home; + } + if (mavenHome == null) { + String maven_home = System.getenv("MAVEN_HOME"); // legacy, for maven 1 + if (StringUtils.isNotEmpty(maven_home)) { + LOGGER.debug("is going to use MAVEN_HOME to read configuration"); + mavenHome = maven_home; + } + } + if (mavenHome == null) { + // try to detect mvn executable from $PATH !! (cf maven distrib bin/mvn.cmd, or bin/mvn: setting M2_HOME if not set) + String path = System.getenv("PATH"); + String pathSep = System.getProperty("path.separator"); + String[] pathElts = path.split(pathSep); + for(String pathElt : pathElts) { + File pathDir = new File(pathElt); + if (new File(pathDir, "mvn").exists()) { + mavenHome = pathDir.getParentFile().getAbsolutePath(); + LOGGER.debug("found mvn in PATH => " + mavenHome); + break; + } + } } return mavenHome; } @@ -254,17 +276,16 @@ public static void main(String[] args) throws BootstrapException { } }); - RepositorySystem system = null; + RepositorySystem repositorySystem; try { - system = getRepositorySystem(); + repositorySystem = getRepositorySystem(); } catch (ComponentLookupException e) { - LOGGER.error("unable to get RepositoySystem from external maven", e); + throw new BootstrapException("unable to get RepositoySystem from external maven", e); } - DefaultRepositorySystemSession session = newRepositorySystemSession(system); + DefaultRepositorySystemSession session = newRepositorySystemSession(repositorySystem); DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.RUNTIME); - RepositorySystem finalSystem = system; - componentsToStart.stream().forEach(c -> { + for(Component c : componentsToStart) { Artifact artifact = new DefaultArtifact(hadoopUnitConfiguration.getString(c.getArtifactKey())); CollectRequest collectRequest = new CollectRequest(); collectRequest.setRoot(new Dependency(artifact, JavaScopes.RUNTIME)); @@ -272,11 +293,12 @@ public static void main(String[] args) throws BootstrapException { DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, classpathFilter); - List artifactResults = null; + List artifactResults; try { - artifactResults = finalSystem.resolveDependencies(session, dependencyRequest).getArtifactResults(); + artifactResults = repositorySystem.resolveDependencies(session, dependencyRequest).getArtifactResults(); } catch (DependencyResolutionException e) { - e.printStackTrace(); + LOGGER.error("Failed to resolve dependency artifact " + artifact + " .. ignore from classpath!", e); + continue; } List artifacts = new ArrayList<>(); @@ -287,7 +309,7 @@ public static void main(String[] args) throws BootstrapException { componentsProperty.add(componentProperties); componentsToStop.add(0, componentProperties); - }); + } Runtime.getRuntime().addShutdownHook(new Thread() { @@ -310,10 +332,11 @@ public void run() { } private static List getRemoteRepositories() { - if (StringUtils.isEmpty(getInstalledMavenHome())) { + String mavenHome = getInstalledMavenHome(); + if (StringUtils.isEmpty(mavenHome)) { return newRepositories(); } else { - List mirrors = getLocalSettings(getInstalledMavenHome()).getMirrors(); + List mirrors = getLocalSettings(mavenHome).getMirrors(); List remoteRepositories = mirrors.stream().map(mirror -> new RemoteRepository.Builder(mirror.getId(), "default", mirror.getUrl()).build()).collect(Collectors.toList()); return remoteRepositories; } @@ -341,8 +364,9 @@ private static void printBanner() { System.out.println(); } - private static ComponentProperties loadAndRun(String c, String className, List artifacts) { - List urls = new ArrayList(); + @SuppressWarnings("resource") + private static ComponentProperties loadAndRun(String c, String className, List artifacts) { + List urls = new ArrayList<>(); urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("log4j.xml")); urls.add(HadoopStandaloneBootstrap.class.getClassLoader().getResource("logback.xml")); @@ -364,20 +388,20 @@ private static ComponentProperties loadAndRun(String c, String className, List mainClass; try { mainClass = classloader.loadClass(className); } catch (ClassNotFoundException e) { LOGGER.error("unable to load class", e); + return null; } - Method main = null; try { Thread.currentThread().setContextClassLoader(classloader); Object o = mainClass.getConstructor(URL.class).newInstance(HadoopStandaloneBootstrap.class.getClassLoader().getResource(DEFAULT_PROPS_FILE)); - main = mainClass.getMethod("start"); + Method main = mainClass.getMethod("start"); main.invoke(o); return new ComponentProperties(o, mainClass); } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { @@ -388,9 +412,9 @@ private static ComponentProperties loadAndRun(String c, String className, List mainClass; - public ComponentProperties(Object instance, Class mainClass) { + public ComponentProperties(Object instance, Class mainClass) { this.instance = instance; this.mainClass = mainClass; } @@ -399,7 +423,7 @@ public Object getInstance() { return instance; } - public Class getMainClass() { + public Class getMainClass() { return mainClass; } } diff --git a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties index 490d3489..0f2a6e94 100644 --- a/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties +++ b/hadoop-unit-standalone/src/main/resources/hadoop-unit-default.properties @@ -27,7 +27,7 @@ confluent.ksql_rest.artifact=fr.jetoile.hadoop:hadoop-unit-confluent:2.10-SNAPSH confluent.kafka_rest.artifact=fr.jetoile.hadoop:hadoop-unit-confluent-rest:2.10-SNAPSHOT maven.central.repo=https://repo.maven.apache.org/maven2/ -maven.local.repo=/home/khanh/.m2/repository +maven.local.repo=~/.m2/repository maven.debug=true @@ -139,7 +139,7 @@ oozie.hdfs.share.lib.dir=/tmp/share_lib oozie.share.lib.create=true oozie.local.share.lib.cache.dir=/tmp/share_lib_cache oozie.purge.local.share.lib.cache=false -oozie.sharelib.path=/home/khanh/github +oozie.sharelib.path=~/github oozie.sharelib.name=oozie-4.2.0.2.6.3.0-235-distro.tar.gz oozie.port=20113 oozie.host=localhost diff --git a/hadoop-unit-standalone/test-console.sh b/hadoop-unit-standalone/test-console.sh new file mode 100755 index 00000000..c91dc00a --- /dev/null +++ b/hadoop-unit-standalone/test-console.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd target + +mkdir -p test-run +cd test-run + +tar zxf ../hadoop-unit-standalone*.tar.gz +cd hadoop-unit-standalone* + +./bin/hadoop-unit-standalone console + From a5ed0b575127618d99d76de94c7d3f6b29eb8f09 Mon Sep 17 00:00:00 2001 From: jetoile Date: Thu, 18 Oct 2018 11:04:15 +0200 Subject: [PATCH 2/3] do not use PATH to find maven --- .../hadoopunit/HadoopStandaloneBootstrap.java | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java b/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java index b70d59e4..6bf88edb 100644 --- a/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java +++ b/hadoop-unit-standalone/src/main/java/fr/jetoile/hadoopunit/HadoopStandaloneBootstrap.java @@ -133,7 +133,7 @@ public void serviceCreationFailed(Class type, Class impl, Throwable except return locator.getService(RepositorySystem.class); } - public static DefaultRepositorySystemSession newRepositorySystemSession(RepositorySystem system) { + public static DefaultRepositorySystemSession newRepositorySystemSession(RepositorySystem system) throws BootstrapException { DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession(); String localRepositoryDir = ""; @@ -158,8 +158,7 @@ public static DefaultRepositorySystemSession newRepositorySystemSession(Reposito LOGGER.debug("is going to use the maven repository from {} with key {}", DEFAULT_PROPS_FILE, "maven.local.repo"); localRepositoryDir = HadoopUtils.resolveDir(localRepositoryDir); } else { - // using default - localRepositoryDir = userHome + "/.m2/repository"; + throw new BootstrapException("unable to find M2_HOME/MAVEN_HOME or the configuration key maven.local.repo from "+ DEFAULT_PROPS_FILE); } } @@ -221,20 +220,6 @@ private static String getInstalledMavenHome() { mavenHome = maven_home; } } - if (mavenHome == null) { - // try to detect mvn executable from $PATH !! (cf maven distrib bin/mvn.cmd, or bin/mvn: setting M2_HOME if not set) - String path = System.getenv("PATH"); - String pathSep = System.getProperty("path.separator"); - String[] pathElts = path.split(pathSep); - for(String pathElt : pathElts) { - File pathDir = new File(pathElt); - if (new File(pathDir, "mvn").exists()) { - mavenHome = pathDir.getParentFile().getAbsolutePath(); - LOGGER.debug("found mvn in PATH => " + mavenHome); - break; - } - } - } return mavenHome; } @@ -297,8 +282,7 @@ public static void main(String[] args) throws BootstrapException { try { artifactResults = repositorySystem.resolveDependencies(session, dependencyRequest).getArtifactResults(); } catch (DependencyResolutionException e) { - LOGGER.error("Failed to resolve dependency artifact " + artifact + " .. ignore from classpath!", e); - continue; + throw new BootstrapException("failed to resolve dependency artifact " + artifact, e); } List artifacts = new ArrayList<>(); From 952155e223528281c873be366dfc3a713b138021 Mon Sep 17 00:00:00 2001 From: jetoile Date: Thu, 18 Oct 2018 11:05:33 +0200 Subject: [PATCH 3/3] remove script test-console.sh --- hadoop-unit-standalone/test-console.sh | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100755 hadoop-unit-standalone/test-console.sh diff --git a/hadoop-unit-standalone/test-console.sh b/hadoop-unit-standalone/test-console.sh deleted file mode 100755 index c91dc00a..00000000 --- a/hadoop-unit-standalone/test-console.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -cd target - -mkdir -p test-run -cd test-run - -tar zxf ../hadoop-unit-standalone*.tar.gz -cd hadoop-unit-standalone* - -./bin/hadoop-unit-standalone console -