embedded dll for windows users

hadoop-unit-commons/pom.xml

@@ -19,6 +19,11 @@
             <artifactId>commons-configuration</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>fr.jetoile.hadoop</groupId>
+            <artifactId>hadoop-unit-windowslibs</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>

hadoop-unit-commons/src/main/java/fr/jetoile/hadoopunit/HadoopUtils.java

@@ -16,7 +16,9 @@
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,48 +28,81 @@
 public enum HadoopUtils {
     INSTANCE;
 
-    private final Logger LOG = LoggerFactory.getLogger(HadoopUtils.class);
+    public static final String WINUTILS_EXE = "winutils.exe";
+    public static final String HADOOP_HOME = "HADOOP_HOME";
+    public static final String HADOOP_DLL = "/hadoop.dll";
+    public static final String HDFS_DLL = "/hdfs.dll";
+
+    private final Logger LOGGER = LoggerFactory.getLogger(HadoopUtils.class);
     private Configuration configuration;
 
     private HadoopUtils() {
         // Set hadoop.home.dir to point to the windows lib dir
         if (System.getProperty("os.name").startsWith("Windows")) {
 
-            if (StringUtils.isEmpty(System.getenv("HADOOP_HOME"))) {
+            if (StringUtils.isEmpty(System.getenv(HADOOP_HOME))) {
 
                 try {
                     configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
                 } catch (ConfigurationException e) {
-                    LOG.error("unable to load {}", HadoopUnitConfig.DEFAULT_PROPS_FILE, e);
+                    LOGGER.error("unable to load {}", HadoopUnitConfig.DEFAULT_PROPS_FILE, e);
                 }
 
-                String hadoop_home = configuration.getString("HADOOP_HOME");
+                String hadoop_home = configuration.getString(HADOOP_HOME);
 
-                LOG.info("Setting hadoop.home.dir: {}", hadoop_home);
+                LOGGER.info("Setting hadoop.home.dir: {}", hadoop_home);
                 if (hadoop_home == null) {
-                    LOG.error("HADOOP_HOME should be set or informed into hadoop-unit-default.properties");
+                    LOGGER.error("HADOOP_HOME should be set or informed into hadoop-unit-default.properties");
                     System.exit(-1);
                 } else {
-                    System.setProperty("HADOOP_HOME", hadoop_home);
+                    System.setProperty(HADOOP_HOME, hadoop_home);
                 }
 
             } else {
-                System.setProperty("HADOOP_HOME", System.getenv("HADOOP_HOME"));
+                System.setProperty(HADOOP_HOME, System.getenv(HADOOP_HOME));
             }
 
-            String windowsLibDir = System.getenv("HADOOP_HOME");
+            String windowsLibDir = System.getenv(HADOOP_HOME);
 
-            LOG.info("WINDOWS: Setting hadoop.home.dir: {}", windowsLibDir);
-            System.setProperty("hadoop.home.dir", windowsLibDir);
             try {
-                System.load(new File(windowsLibDir + Path.SEPARATOR + "bin" + Path.SEPARATOR + "hadoop.dll").getAbsolutePath());
-                System.load(new File(windowsLibDir + Path.SEPARATOR + "bin" + Path.SEPARATOR + "hdfs.dll").getAbsolutePath());
-            } catch (UnsatisfiedLinkError e) {
-                //ignore it
+                extractAndMoveWinUtils(windowsLibDir);
+                extractAndLoadDll(HADOOP_DLL);
+                extractAndLoadDll(HDFS_DLL);
+            } catch (IOException e) {
+                LOGGER.error("unable to load windows dll", e);
             }
         }
     }
 
+    private void extractAndLoadDll(String lib) throws IOException {
+        InputStream in = HadoopUtils.class.getResourceAsStream(lib);
+        // always write to different location
+        File fileOut = new File(System.getProperty("java.io.tmpdir") + Path.SEPARATOR + lib);
+        System.out.println("Writing dll to: " + fileOut.getAbsolutePath());
+
+        OutputStream out = FileUtils.openOutputStream(fileOut);
+        IOUtils.copy(in, out);
+        in.close();
+        out.close();
+        try {
+            System.load(fileOut.getAbsolutePath());
+        } catch (UnsatisfiedLinkError e) {
+            //ignore it
+        }
+    }
+
+    private void extractAndMoveWinUtils(String path) throws IOException {
+        InputStream in = HadoopUtils.class.getResourceAsStream("/" + WINUTILS_EXE);
+        // always write to different location
+        File fileOut = new File(path + Path.SEPARATOR + "bin" + Path.SEPARATOR + WINUTILS_EXE);
+        LOGGER.info("Writing {} to: {}", WINUTILS_EXE, fileOut.getAbsolutePath());
+
+        OutputStream out = FileUtils.openOutputStream(fileOut);
+        IOUtils.copy(in, out);
+        in.close();
+        out.close();
+    }
+
     public void setHadoopHome() {
 
     }
@@ -81,9 +116,8 @@ public void printBanner(PrintStream out) {
             while ((line = br.readLine()) != null) {
                 out.println(line);
             }
-        }
-        catch (Exception ex) {
-            LOG.warn("Banner not printable", ex);
+        } catch (Exception ex) {
+            LOGGER.warn("Banner not printable", ex);
         }
     }
 }

hadoop-unit-hbase/src/main/java/fr/jetoile/hadoopunit/component/HBaseBootstrap.java

@@ -108,7 +108,6 @@ private void build() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveMetastoreBootstrap.java

@@ -71,7 +71,6 @@ public String getProperties() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-hive/src/main/java/fr/jetoile/hadoopunit/component/HiveServer2Bootstrap.java

@@ -75,7 +75,6 @@ public String getProperties() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-kafka/src/main/java/fr/jetoile/hadoopunit/component/KafkaBootstrap.java

@@ -87,7 +87,6 @@ private void build() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-knox/src/main/java/fr/jetoile/hadoopunit/component/KnoxBootstrap.java

@@ -135,7 +135,6 @@ String getTopology(List<KnoxService> services) {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-oozie/src/main/java/fr/jetoile/hadoopunit/component/OozieBootstrap.java

@@ -158,7 +158,6 @@ private void build() throws NotFoundServiceException {
     }
 
     private void loadConfig() throws BootstrapException, NotFoundServiceException {
-        HadoopUtils.INSTANCE.setHadoopHome();
 
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);

hadoop-unit-solr/src/main/java/fr/jetoile/hadoopunit/component/SolrBootstrap.java

@@ -74,7 +74,6 @@ public String getProperties() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-solrcloud/src/main/java/fr/jetoile/hadoopunit/component/SolrCloudBootstrap.java

@@ -107,7 +107,6 @@ private void build() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

hadoop-unit-windowslibs/pom.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>hadoop-unit</artifactId>
+        <groupId>fr.jetoile.hadoop</groupId>
+        <version>2.2-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>hadoop-unit-windowslibs</artifactId>
+
+
+</project>

hadoop-unit-zookeeper/src/main/java/fr/jetoile/hadoopunit/component/ZookeeperBootstrap.java

@@ -73,7 +73,6 @@ public String getProperties() {
     }
 
     private void loadConfig() throws BootstrapException {
-        HadoopUtils.INSTANCE.setHadoopHome();
         try {
             configuration = new PropertiesConfiguration(HadoopUnitConfig.DEFAULT_PROPS_FILE);
         } catch (ConfigurationException e) {

pom.xml

@@ -65,6 +65,7 @@
         <module>sample</module>
         <module>hadoop-unit-knox</module>
         <module>hadoop-unit-alluxio</module>
+        <module>hadoop-unit-windowslibs</module>
     </modules>
 
     <repositories>
@@ -238,6 +239,12 @@
                 </exclusions>
             </dependency>
 
+            <dependency>
+                <groupId>fr.jetoile.hadoop</groupId>
+                <artifactId>hadoop-unit-windowslibs</artifactId>
+                <version>${hadoop-unit.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>fr.jetoile.hadoop</groupId>
                 <artifactId>hadoop-unit-zookeeper</artifactId>