add knox integration test with gateway-shell

jetoile · Jan 24, 2017 · c4dcb2a · c4dcb2a
1 parent 91599fd
commit c4dcb2a
Show file tree

Hide file tree

Showing 5 changed files with 441 additions and 0 deletions.
diff --git a/sample/knox-hbase-webhdfs/pom.xml b/sample/knox-hbase-webhdfs/pom.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>sample</artifactId>
+        <groupId>fr.jetoile.hadoop</groupId>
+        <version>2.2-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>knox-hbase-webhdfs</artifactId>
+
+    <properties>
+        <gateway-shell.version>0.11.0</gateway-shell.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.knox</groupId>
+            <artifactId>gateway-shell</artifactId>
+            <version>${gateway-shell.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-configuration</groupId>
+            <artifactId>commons-configuration</artifactId>
+        </dependency>
+    </dependencies>
+
+
+    <profiles>
+        <profile>
+            <id>default</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>!travis</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes>
+                                <exclude>**/*IntegrationTest.java</exclude>
+                            </excludes>
+                        </configuration>
+                        <executions>
+                            <execution>
+                                <id>integration-test</id>
+                                <goals>
+                                    <goal>test</goal>
+                                </goals>
+                                <phase>integration-test</phase>
+                                <configuration>
+                                    <excludes>
+                                        <exclude>none</exclude>
+                                    </excludes>
+                                    <includes>
+                                        <include>**/*IntegrationTest.java</include>
+                                    </includes>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
+                    <plugin>
+                        <artifactId>hadoop-unit-maven-plugin</artifactId>
+                        <groupId>fr.jetoile.hadoop</groupId>
+                        <version>${hadoop-unit.version}</version>
+                        <executions>
+                            <execution>
+                                <id>start</id>
+                                <goals>
+                                    <goal>embedded-start</goal>
+                                </goals>
+                                <phase>pre-integration-test</phase>
+                            </execution>
+                        </executions>
+                        <configuration>
+                            <components>
+                                <componentArtifact implementation="fr.jetoile.hadoopunit.ComponentArtifact">
+                                    <componentName>ZOOKEEPER</componentName>
+                                    <artifact>fr.jetoile.hadoop:hadoop-unit-zookeeper:${hadoop-unit.version}</artifact>
+                                </componentArtifact>
+                                <componentArtifact implementation="fr.jetoile.hadoopunit.ComponentArtifact">
+                                    <componentName>HDFS</componentName>
+                                    <artifact>fr.jetoile.hadoop:hadoop-unit-hdfs:${hadoop-unit.version}</artifact>
+                                </componentArtifact>
+                                <componentArtifact implementation="fr.jetoile.hadoopunit.ComponentArtifact">
+                                    <componentName>HBASE</componentName>
+                                    <artifact>fr.jetoile.hadoop:hadoop-unit-hbase:${hadoop-unit.version}</artifact>
+                                </componentArtifact>
+                                <componentArtifact implementation="fr.jetoile.hadoopunit.ComponentArtifact">
+                                    <componentName>KNOX</componentName>
+                                    <artifact>fr.jetoile.hadoop:hadoop-unit-knox:${hadoop-unit.version}</artifact>
+                                </componentArtifact>
+                            </components>
+
+                        </configuration>
+
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+
+        <profile>
+            <id>travis</id>
+            <activation>
+                <activeByDefault>false</activeByDefault>
+                <property>
+                    <name>travis</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <skipTests>true</skipTests>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>
diff --git a/sample/knox-hbase-webhdfs/src/main/java/fr/jetoile/hadoopunit/sample/KnoxJob.java b/sample/knox-hbase-webhdfs/src/main/java/fr/jetoile/hadoopunit/sample/KnoxJob.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package fr.jetoile.hadoopunit.sample;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.hadoop.gateway.shell.hbase.HBase;
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+
+public class KnoxJob {
+
+    public void createHdfsDirectory(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        Hdfs.mkdir(hadoop).dir("/hdfs/test").now();
+    }
+
+    public void createFileOnHdfs(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        Hdfs.put(hadoop).text("TEST").to("/hdfs/test/sample.txt").now();
+    }
+
+    public String getFileOnHdfs(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        InputStream inputStream = Hdfs.get(hadoop).from("/hdfs/test/sample.txt").now().getStream();
+        return IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+    }
+
+    public String getHBaseStatus(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        InputStream inputStream = HBase.session(hadoop).status().now().getStream();
+        return IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+    }
+
+    public void createHBaseTable(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        HBase.session(hadoop).table("test").create().family("family1").endFamilyDef().now();
+    }
+
+
+    public String getHBaseTableSchema(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        InputStream inputStream = HBase.session(hadoop).table("test").schema().now().getStream();
+        return IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+    }
+
+
+    public void putHBaseData(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        HBase.session(hadoop).table("test").row("row_id_1").store()
+                .column("family1", "col1", "col_value1")
+                .now();
+    }
+
+    public String readHBaseData(Hadoop hadoop) throws ConfigurationException, URISyntaxException, KeyManagementException, NoSuchAlgorithmException, IOException {
+        InputStream inputStream = HBase.session(hadoop).table("test").row("row_id_1")
+                .query()
+                .now().getStream();
+        return IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+    }
+
+}
diff --git a/...knox-hbase-webhdfs/src/test/java/fr/jetoile/hadoopunit/sample/KnoxJobIntegrationTest.java b/...knox-hbase-webhdfs/src/test/java/fr/jetoile/hadoopunit/sample/KnoxJobIntegrationTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package fr.jetoile.hadoopunit.sample;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+
+import static org.junit.Assert.assertTrue;
+
+public class KnoxJobIntegrationTest {
+
+    private static Hadoop hadoop;
+
+    @BeforeClass
+    static public void setUp() throws ConfigurationException, URISyntaxException {
+        Configuration configuration = new PropertiesConfiguration("src/test/resources/hadoop-unit-default.properties");
+
+        String host = configuration.getString("knox.host");
+        String port = configuration.getString("knox.port");
+        String gateway = configuration.getString("knox.path");
+        String cluster = configuration.getString("knox.cluster");
+
+        hadoop = Hadoop.loginInsecure("https://" + host + ":" + port + "/" + gateway + "/" + cluster, "none", "none");
+    }
+
+    @AfterClass
+    static public void tearDown() throws InterruptedException {
+        hadoop.shutdown();
+    }
+
+    @Test
+    public void hdfs_through_knox_should_be_ok() throws ConfigurationException, KeyManagementException, URISyntaxException, IOException, NoSuchAlgorithmException {
+        KnoxJob knoxJob = new KnoxJob();
+        knoxJob.createHdfsDirectory(hadoop);
+        knoxJob.createFileOnHdfs(hadoop);
+        knoxJob.getFileOnHdfs(hadoop);
+    }
+
+
+    @Test
+    public void hbase_through_knox_should_be_ok() throws ConfigurationException, KeyManagementException, URISyntaxException, IOException, NoSuchAlgorithmException {
+        KnoxJob knoxJob = new KnoxJob();
+
+        assertTrue(knoxJob.getHBaseStatus(hadoop).contains("\"regions\":3"));
+        knoxJob.createHBaseTable(hadoop);
+
+        System.out.println("==============================");
+        System.out.println(knoxJob.getHBaseTableSchema(hadoop));
+        System.out.println("==============================");
+
+        assertTrue(knoxJob.getHBaseTableSchema(hadoop).contains("{\"name\":\"test\",\"ColumnSchema\":[{\"name\":\"family1\""));
+        knoxJob.putHBaseData(hadoop);
+
+        System.out.println("==============================");
+        System.out.println(knoxJob.readHBaseData(hadoop));
+        System.out.println("==============================");
+
+        assertTrue(knoxJob.readHBaseData(hadoop).contains("{\"Row\":[{\"key\":\""));
+    }
+}