Merge pull request #336 from xitongsys/dev

Dev
xitongsys · Dec 2, 2020 · cb8d885 · cb8d885
2 parents ecff48f + b2cf549
commit cb8d885
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 24 deletions.
diff --git a/tool/parquet-tools/README.md b/tool/parquet-tools/README.md
@@ -36,7 +36,7 @@ parquet_go_root struct{
   "Tag": "name=parquet_go_root, repetitiontype=REQUIRED",
   "Fields": [
     {
-      "Tag": "name=name, type=UTF8, repetitiontype=REQUIRED",
+      "Tag": "name=name, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=REQUIRED",
       "Fields": null
     },
     {

diff --git a/tool/parquet-tools/schematool/schematool.go b/tool/parquet-tools/schematool/schematool.go
@@ -96,18 +96,6 @@ func ParquetTypeToGoTypeStr(pT *parquet.Type, cT *parquet.ConvertedType) string
 			res = "string"
 		}
 	}
-	if cT != nil {
-		switch *cT {
-		case parquet.ConvertedType_UINT_8:
-			res = "uint32"
-		case parquet.ConvertedType_UINT_16:
-			res = "uint32"
-		case parquet.ConvertedType_UINT_32:
-			res = "uint32"
-		case parquet.ConvertedType_UINT_64:
-			res = "uint64"
-		}
-	}
 	return res
 }
 
@@ -138,6 +126,7 @@ func (self *Node) OutputJsonSchema() string {
 
 	pTStr, cTStr := ParquetTypeToParquetTypeStr(pT, cT)
 	tagStr := "\"name=%s, type=%s, repetitiontype=%s\""
+
 	name := self.SE.GetName()
 
 	if len(self.Children) == 0 {
@@ -150,19 +139,21 @@ func (self *Node) OutputJsonSchema() string {
 			scale, precision := self.SE.GetScale(), self.SE.GetPrecision()
 			if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY {
 				length := self.SE.GetTypeLength()
-				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%d, length=%d, repetitiontype=%s\""
-				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, length, rTStr) + "}"
+				tagStr = "\"name=%s, type=%s, convertedtype=%s, scale=%d, precision=%d, length=%d, repetitiontype=%s\""
+				res += fmt.Sprintf(tagStr, name, pTStr, cTStr, scale, precision, length, rTStr) + "}"
 			} else {
-				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%d, repetitiontype\""
-				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, rTStr) + "}"
+				tagStr = "\"name=%s, type=%s, convertedtype=%s, scale=%d, precision=%d, repetitiontype\""
+				res += fmt.Sprintf(tagStr, name, pTStr, cTStr, scale, precision, rTStr) + "}"
 			}
 
 		} else {
-			typeStr := pTStr
 			if cT != nil {
-				typeStr = cTStr
+				tagStr := "\"name=%s, type=%s, convertedtype=%s, repetitiontype=%s\""
+				res += fmt.Sprintf(tagStr, name, pTStr, cTStr, rTStr) + "}"
+
+			} else {
+				res += fmt.Sprintf(tagStr, name, pTStr, rTStr) + "}"
 			}
-			res += fmt.Sprintf(tagStr, name, typeStr, rTStr) + "}"
 
 		}
 	} else {
@@ -247,11 +238,11 @@ func (self *Node) getStructTags() string {
 		scale, precision := self.SE.GetScale(), self.SE.GetPrecision()
 		if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY {
 			length := self.SE.GetTypeLength()
-			tagStr := "`parquet:\"name=%s, type=%s, basetype=%s, scale=%d, precision=%d, length=%d, repetitiontype=%s\"`"
-			tags = fmt.Sprintf(tagStr, self.SE.Name, cTStr, pTStr, scale, precision, length, rTStr)
+			tagStr := "`parquet:\"name=%s, type=%s, convertedtype=%s, scale=%d, precision=%d, length=%d, repetitiontype=%s\"`"
+			tags = fmt.Sprintf(tagStr, self.SE.Name, pTStr, cTStr, scale, precision, length, rTStr)
 		} else {
-			tagStr := "`parquet:\"name=%s, type=%s, basetype=%s, scale=%d, precision=%d, repetitiontype\"`"
-			tags = fmt.Sprintf(tagStr, self.SE.Name, self.SE.Type, pTStr, scale, precision, rTStr)
+			tagStr := "`parquet:\"name=%s, type=%s, convertedtype=%s, scale=%d, precision=%d, repetitiontype\"`"
+			tags = fmt.Sprintf(tagStr, self.SE.Name, pTStr, self.SE.Type, scale, precision, rTStr)
 		}
 	}