fix: Spark Loader example schema and struct mismatch

apache · Aug 3, 2023 · 5e8322f · 5e8322f
1 parent 549ca82
commit 5e8322f
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 20 deletions.
diff --git a/hugegraph-loader/assembly/static/example/spark/schema.groovy b/hugegraph-loader/assembly/static/example/spark/schema.groovy
@@ -26,13 +26,13 @@ schema.propertyKey("price").asDouble().ifNotExist().create();
 
 schema.vertexLabel("person")
       .properties("name", "age", "city")
-      .primaryKeys("name")
+      .useCustomizeStringId()
       .nullableKeys("age", "city")
       .ifNotExist()
       .create();
 schema.vertexLabel("software")
-        .properties("name", "lang", "price")
-        .primaryKeys("name")
+      .properties("name", "lang", "price")
+      .useCustomizeStringId()
       .ifNotExist()
       .create();
 

diff --git a/hugegraph-loader/assembly/static/example/spark/struct-hbase.json b/hugegraph-loader/assembly/static/example/spark/struct-hbase.json
@@ -0,0 +1,56 @@
+{
+  "vertices": [
+    {
+      "label": "person",
+      "input": {
+        "type": "file",
+        "path": "example/spark/vertex_person.json",
+        "format": "JSON",
+        "header": ["name", "age", "city"],
+        "charset": "UTF-8",
+        "skipped_line": {
+          "regex": "(^#|^//).*"
+        }
+      },
+      "id": "name",
+      "null_values": ["NULL", "null", ""]
+    },
+    {
+      "label": "software",
+      "input": {
+        "type": "file",
+        "path": "example/spark/vertex_software.json",
+        "format": "JSON",
+        "header": ["id","name", "lang", "price","ISBN"],
+        "charset": "GBK"
+      },
+      "id": "name",
+      "ignored": ["ISBN"]
+    }
+  ],
+  "edges": [
+    {
+      "label": "knows",
+      "source": ["source_name"],
+      "target": ["target_name"],
+      "input": {
+        "type": "file",
+        "path": "example/spark/edge_knows.json",
+        "format": "JSON",
+        "date_format": "yyyyMMdd",
+        "header": ["source_name","target_name", "date", "weight"]
+      },
+      "field_mapping": {
+        "source_name": "name",
+        "target_name": "name"
+      }
+    }
+  ],
+  "backendStoreInfo": {
+    "edge_tablename": "hugegraph:g_oe",
+    "vertex_tablename": "hugegraph:g_v",
+    "hbase_zookeeper_quorum": "127.0.0.1",
+    "hbase_zookeeper_property_clientPort": "2181",
+    "zookeeper_znode_parent": "/hbase"
+  }
+}
diff --git a/hugegraph-loader/assembly/static/example/spark/struct.json b/hugegraph-loader/assembly/static/example/spark/struct.json
@@ -45,13 +45,5 @@
         "target_name": "name"
       }
     }
-  ],
-  "backendStoreInfo":
-  {
-    "edge_tablename": "hugegraph:g_oe",
-    "vertex_tablename": "hugegraph:g_v",
-    "hbase_zookeeper_quorum": "127.0.0.1",
-    "hbase_zookeeper_property_clientPort": "2181",
-    "zookeeper_znode_parent": "/hbase"
-  }
+  ]
 }
diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_person.json b/hugegraph-loader/assembly/static/example/spark/vertex_person.json
@@ -1,6 +1,6 @@
-{"name": "marko", "age": "29",  "city": "Beijing"}
-{"name": "vadas", "age": "27",  "city": "Hongkong"}
-{"name": "josh", "age": "32",  "city": "Beijing"}
-{"name": "peter", "age": "35",  "city": "Shanghai"}
-{"name": "li,nary", "age": "26",  "city": "Wu,han"}
-{"name": "tom", "age": "null",  "city": "NULL"}
+{"name": "marko", "age": "29", "city": "Beijing"}
+{"name": "vadas", "age": "27", "city": "Hongkong"}
+{"name": "josh", "age": "32", "city": "Beijing"}
+{"name": "peter", "age": "35", "city": "Shanghai"}
+{"name": "li,nary", "age": "26", "city": "Wu,han"}
+{"name": "tom", "age": "null", "city": "NULL"}
diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_software.json b/hugegraph-loader/assembly/static/example/spark/vertex_software.json
@@ -1,2 +1,2 @@
-{ "name": "lop",  "lang": "java","price": "328","ISBN": "ISBN978-7-107-18618-5"}
-{ "name": "ripple",  "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"}
+{"name": "lop", "lang": "java", "price": "328", "ISBN": "ISBN978-7-107-18618-5"}
+{"name": "ripple", "lang": "java", "price": "199", "ISBN": "ISBN978-7-100-13678-5"}