feat(hbase): support gen HFile for hbase v2 (BETA)

hugegraph-dist/scripts/dependency/known-dependencies.txt

@@ -179,6 +179,24 @@ hadoop-yarn-client-3.1.1.jar
 hadoop-yarn-common-3.1.1.jar
 hamcrest-2.1.jar
 hamcrest-core-1.3.jar
+hbase-client-2.2.3.jar
+hbase-common-2.2.3.jar
+hbase-hadoop-compat-2.2.3.jar
+hbase-hadoop2-compat-2.2.3.jar
+hbase-http-2.2.3.jar
+hbase-mapreduce-2.2.3.jar
+hbase-metrics-2.2.3.jar
+hbase-metrics-api-2.2.3.jar
+hbase-procedure-2.2.3.jar
+hbase-protocol-2.2.3.jar
+hbase-protocol-shaded-2.2.3.jar
+hbase-replication-2.2.3.jar
+hbase-server-2.2.3.jar
+hbase-shaded-client-byo-hadoop-2.2.3.jar
+hbase-shaded-miscellaneous-2.2.1.jar
+hbase-shaded-netty-2.2.1.jar
+hbase-shaded-protobuf-2.2.1.jar
+hbase-zookeeper-2.2.3.jar
 hibernate-validator-6.0.17.Final.jar
 hive-classification-3.1.2.jar
 hive-classification-3.1.3.jar
@@ -225,6 +243,7 @@ hk2-utils-3.0.1.jar
 hppc-0.7.2.jar
 htrace-core4-4.0.1-incubating.jar
 htrace-core4-4.1.0-incubating.jar
+htrace-core4-4.2.0-incubating.jar
 httpclient-4.5.13.jar
 httpclient-4.5.2.jar
 httpclient-4.5.9.jar
@@ -579,3 +598,13 @@ zookeeper-3.4.9.jar
 zookeeper-3.6.2.jar
 zookeeper-jute-3.6.2.jar
 zstd-jni-1.5.0-4.jar
+disruptor-3.3.6.jar
+findbugs-annotations-1.3.9-1.jar
+jamon-runtime-2.4.1.jar
+javax.el-3.0.1-b12.jar
+javax.servlet.jsp-2.3.2.jar
+javax.servlet.jsp-api-2.3.1.jar
+jcodings-1.0.18.jar
+joni-2.1.11.jar
+libfb303-0.9.3.jar
+parquet-hadoop-bundle-1.10.0.jar

hugegraph-loader/assembly/static/bin/get-params.sh

@@ -27,7 +27,7 @@ function get_params() {
       --incremental-mode | --failure-mode | --batch-insert-threads | --single-insert-threads | \
       --max-conn | --max-conn-per-route | --batch-size | --max-parse-errors | --max-insert-errors | \
       --timeout | --shutdown-timeout | --retry-times | --retry-interval | --check-vertex | \
-      --print-progress | --dry-run | --help)
+      --print-progress | --dry-run | --sink-type | --vertex-partitions | --edge-partitions | --help )
         HUGEGRAPH_PARAMS="$HUGEGRAPH_PARAMS $1 $2"
         shift 2
         ;;

hugegraph-loader/assembly/static/example/spark/edge_created.json

@@ -0,0 +1,4 @@
+{"source_name": "marko", "target_id": 1, "date": "2017-12-10", "weight": 0.4}
+{"source_name": "josh", "target_id": 1, "date": "2009-11-11", "weight": 0.4}
+{"source_name": "josh", "target_id": 2, "date": "2017-12-10", "weight": 1.0}
+{"source_name": "peter", "target_id": 1, "date": "2017-03-24", "weight": 0.2}

hugegraph-loader/assembly/static/example/spark/edge_knows.json

@@ -0,0 +1,2 @@
+{"source_name": "marko", "target_name": "vadas", "date": "20160110", "weight": 0.5}
+{"source_name": "marko", "target_name": "josh", "date": "20130220", "weight": 1.0}

hugegraph-loader/assembly/static/example/spark/schema.groovy

@@ -0,0 +1,36 @@
+// Define schema
+schema.propertyKey("name").asText().ifNotExist().create();
+schema.propertyKey("age").asInt().ifNotExist().create();
+schema.propertyKey("city").asText().ifNotExist().create();
+schema.propertyKey("weight").asDouble().ifNotExist().create();
+schema.propertyKey("lang").asText().ifNotExist().create();
+schema.propertyKey("date").asText().ifNotExist().create();
+schema.propertyKey("price").asDouble().ifNotExist().create();
+
+schema.vertexLabel("person")
+      .properties("name", "age", "city")
+      .primaryKeys("name")
+      .nullableKeys("age", "city")
+      .ifNotExist()
+      .create();
+schema.vertexLabel("software")
+        .properties("name", "lang", "price")
+        .primaryKeys("name")
+      .ifNotExist()
+      .create();
+
+
+schema.edgeLabel("knows")
+      .sourceLabel("person")
+      .targetLabel("person")
+      .properties("date", "weight")
+      .ifNotExist()
+      .create();
+schema.edgeLabel("created")
+      .sourceLabel("person")
+      .targetLabel("software")
+      .properties("date", "weight")
+      .ifNotExist()
+      .create();
+
+

hugegraph-loader/assembly/static/example/spark/struct.json

@@ -0,0 +1,58 @@
+{
+  "vertices": [
+    {
+      "label": "person",
+      "input": {
+        "type": "file",
+        "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/vertex_person.json",
+        "format": "JSON",
+        "header": ["name", "age", "city"],
+        "charset": "UTF-8",
+        "skipped_line": {
+          "regex": "(^#|^//).*"
+        }
+      },
+      "id": "name",
+      "null_values": ["NULL", "null", ""]
+    },
+    {
+      "label": "software",
+      "input": {
+        "type": "file",
+        "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/vertex_software.json",
+        "format": "JSON",
+        "header": ["id","name", "lang", "price","ISBN"],
+        "charset": "GBK"
+      },
+      "id": "name",
+      "ignored": ["ISBN"]
+    }
+  ],
+  "edges": [
+    {
+      "label": "knows",
+      "source": ["source_name"],
+      "target": ["target_name"],
+      "input": {
+        "type": "file",
+        "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/edge_knows.json",
+        "format": "JSON",
+        "date_format": "yyyyMMdd",
+        "header": ["source_name","target_name", "date", "weight"]
+      },
+      "field_mapping": {
+        "source_name": "name",
+        "target_name": "name"
+      }
+    }
+  ],
+  "backendStoreInfo":
+  {
+    "edge_tablename": "hugegraph:g_oe",
+    "vertex_tablename": "hugegraph:g_v",
+    "hbase_zookeeper_quorum":"127.0.0.1",
+    "hbase_zookeeper_property_clientPort":"2181",
+    "zookeeper_znode_parent":"/hbase"
+
+  }
+}

hugegraph-loader/assembly/static/example/spark/vertex_person.json

@@ -0,0 +1,7 @@
+{"name": "marko", "age": "29",  "city": "Beijing"}
+{"name": "vadas", "age": "27",  "city": "Hongkong"}
+{"name": "josh", "age": "32",  "city": "Beijing"}
+{"name": "peter", "age": "35",  "city": "Shanghai"}
+{"name": "li,nary", "age": "26",  "city": "Wu,han"}
+{"name": "tom", "age": "null",  "city": "NULL"}
+

hugegraph-loader/assembly/static/example/spark/vertex_software.json

@@ -0,0 +1,3 @@
+{ "name": "lop",  "lang": "java","price": "328","ISBN": "ISBN978-7-107-18618-5"}
+{ "name": "ripple",  "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"}
+

hugegraph-loader/pom.xml

@@ -142,6 +142,117 @@
             <version>30.0-jre</version>
         </dependency>
 
+        <!--hbase -->
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-common</artifactId>
+            <version>${hbase.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.github.stephenc.findbugs</groupId>
+                    <artifactId>findbugs-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-json</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>${hbase.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>hadoop-auth</artifactId>
+                    <groupId>org.apache.hadoop</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+            <!--<artifactId>hbase-client</artifactId>-->
+            <version>${hbase.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-ipc</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-mapreduce</artifactId>
+            <version>${hbase.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-mapreduce-client-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs</artifactId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>hadoop-auth</artifactId>
+                    <groupId>org.apache.hadoop</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>javax.ws.rs-api</artifactId>
+                    <groupId>javax.ws.rs</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jersey-client</artifactId>
+                    <groupId>org.glassfish.jersey.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jersey-common</artifactId>
+                    <groupId>org.glassfish.jersey.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jersey-server</artifactId>
+                    <groupId>org.glassfish.jersey.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-distcp</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <!-- hadoop dependency -->
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -275,6 +386,10 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-hdfs</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-distcp</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-registry</artifactId>
@@ -316,24 +431,32 @@
                     <artifactId>jersey-json</artifactId>
                 </exclusion>
                 <exclusion>
-                    <groupId>com.sun.jersey.contribs</groupId>
-                    <artifactId>jersey-guice</artifactId>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
                 </exclusion>
                 <exclusion>
-                    <groupId>org.apache.parquet</groupId>
-                    <artifactId>parquet-hadoop-bundle</artifactId>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
                 </exclusion>
                 <exclusion>
-                    <groupId>org.apache.parquet</groupId>
-                    <artifactId>parquet-column</artifactId>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-mapreduce-client-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs</artifactId>
                 </exclusion>
                 <exclusion>
-                    <groupId>org.apache.thrift</groupId>
-                    <artifactId>libfb303</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>*</artifactId>
+                    <artifactId>jackson-core</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>