allow to set datanode address

jetoile · Mar 13, 2019 · ea3453d · ea3453d
1 parent 63493b9
commit ea3453d
Show file tree

Hide file tree

Showing 8 changed files with 2,979 additions and 27 deletions.
diff --git a/...nt-commons/src/main/java/fr/jetoile/hadoopunit/client/commons/HadoopUnitClientConfig.java b/...nt-commons/src/main/java/fr/jetoile/hadoopunit/client/commons/HadoopUnitClientConfig.java
@@ -92,6 +92,9 @@ public class HadoopUnitClientConfig {
     public static final String HDFS_FORMAT_KEY = "hdfs.format";
     public static final String HDFS_ENABLE_RUNNING_USER_AS_PROXY_USER = "hdfs.enable.running.user.as.proxy.user";
     public static final String HDFS_REPLICATION_KEY = "hdfs.replication";
+    public static final String HDFS_DATANODE_ADDRESS_KEY = "hdfs.datanode.address";
+    public static final String HDFS_DATANODE_HTTP_ADDRESS_KEY = "hdfs.datanode.http.address";
+    public static final String HDFS_DATANODE_IPC_ADDRESS_KEY = "hdfs.datanode.ipc.address";
 
     // HDFS Test
     public static final String HDFS_TEST_FILE_KEY = "hdfs.test.file";

diff --git a/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsBootstrap.java b/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsBootstrap.java
@@ -46,6 +46,10 @@ public class HdfsBootstrap implements BootstrapHadoop {
     private String host;
     private Integer replication = 1;
 
+    private String datanodeAddress = "127.0.0.1:50010";
+    private String datanodeHttpAddress = "127.0.0.1:50075";
+    private String datanodeIpcAddress = "127.0.0.1:50020";
+
     public HdfsBootstrap() {
         if (hdfsLocalCluster == null) {
             try {
@@ -73,6 +77,9 @@ public HdfsBootstrap(URL url) {
     public String getProperties() {
         return "\n \t\t\t host:" + host +
                 "\n \t\t\t port:" + port +
+                "\n \t\t\t datanodeAddress:" + datanodeAddress +
+                "\n \t\t\t datanodeHttpAddress:" + datanodeHttpAddress +
+                "\n \t\t\t datanodeIpcAddress:" + datanodeIpcAddress +
                 "\n \t\t\t httpPort:" + httpPort;
     }
 
@@ -85,6 +92,14 @@ private void build() {
         HdfsConfiguration hdfsConfiguration = new HdfsConfiguration();
         hdfsConfiguration.set("dfs.replication", replication.toString());
 
+        hdfsConfiguration.set("dfs.replication", replication.toString());
+        hdfsConfiguration.set("dfs.replication", replication.toString());
+        hdfsConfiguration.set("dfs.replication", replication.toString());
+
+        hdfsConfiguration.set("dfs.datanode.address", datanodeAddress);
+        hdfsConfiguration.set("dfs.datanode.http.address", datanodeHttpAddress);
+        hdfsConfiguration.set("dfs.datanode.ipc.address", datanodeIpcAddress);
+
         hdfsLocalCluster = new HdfsLocalCluster.Builder()
                 .setHdfsNamenodePort(port)
                 .setHdfsNamenodeHttpPort(httpPort)
@@ -108,7 +123,11 @@ private void loadConfig() throws BootstrapException {
         enablePermission = configuration.getBoolean(HdfsConfig.HDFS_ENABLE_PERMISSIONS_KEY);
         format = configuration.getBoolean(HdfsConfig.HDFS_FORMAT_KEY);
         enableRunningUserAsProxy = configuration.getBoolean(HdfsConfig.HDFS_ENABLE_RUNNING_USER_AS_PROXY_USER);
-        replication = configuration.getInt(HdfsConfig.HDFS_REPLICATION_KEY, 1);
+        replication = configuration.getInt(HdfsConfig.HDFS_REPLICATION_KEY, replication);
+
+        datanodeAddress = configuration.getString(HdfsConfig.HDFS_DATANODE_ADDRESS_KEY, datanodeAddress);
+        datanodeHttpAddress = configuration.getString(HdfsConfig.HDFS_DATANODE_HTTP_ADDRESS_KEY, datanodeHttpAddress);
+        datanodeIpcAddress = configuration.getString(HdfsConfig.HDFS_DATANODE_IPC_ADDRESS_KEY, datanodeIpcAddress);
     }
 
     @Override
@@ -140,6 +159,16 @@ public void loadConfig(Map<String, String> configs) {
         if (StringUtils.isNotEmpty(configs.get(HdfsConfig.HDFS_REPLICATION_KEY))) {
             replication = Integer.parseInt(configs.get(HdfsConfig.HDFS_REPLICATION_KEY));
         }
+
+        if (StringUtils.isNotEmpty(configs.get(HdfsConfig.HDFS_DATANODE_ADDRESS_KEY))) {
+            datanodeAddress = configs.get(HdfsConfig.HDFS_DATANODE_ADDRESS_KEY);
+        }
+        if (StringUtils.isNotEmpty(configs.get(HdfsConfig.HDFS_DATANODE_HTTP_ADDRESS_KEY))) {
+            datanodeHttpAddress = configs.get(HdfsConfig.HDFS_DATANODE_HTTP_ADDRESS_KEY);
+        }
+        if (StringUtils.isNotEmpty(configs.get(HdfsConfig.HDFS_DATANODE_IPC_ADDRESS_KEY))) {
+            datanodeIpcAddress = configs.get(HdfsConfig.HDFS_DATANODE_IPC_ADDRESS_KEY);
+        }
     }
 
 

diff --git a/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsConfig.java b/hadoop-unit-hdfs/src/main/java/fr/jetoile/hadoopunit/component/HdfsConfig.java
@@ -26,6 +26,10 @@ public class HdfsConfig {
     public static final String HDFS_ENABLE_RUNNING_USER_AS_PROXY_USER = "hdfs.enable.running.user.as.proxy.user";
     public static final String HDFS_REPLICATION_KEY = "hdfs.replication";
 
+    public static final String HDFS_DATANODE_ADDRESS_KEY = "hdfs.datanode.address";
+    public static final String HDFS_DATANODE_HTTP_ADDRESS_KEY = "hdfs.datanode.http.address";
+    public static final String HDFS_DATANODE_IPC_ADDRESS_KEY = "hdfs.datanode.ipc.address";
+
     // HDFS Test
     public static final String HDFS_TEST_FILE_KEY = "hdfs.test.file";
     public static final String HDFS_TEST_STRING_KEY = "hdfs.test.string";