[fix](case) fix struct format out files (#37350)

follow up #37349 1. Some of constants are calculated in FE, it should output same format as in BE. Add `FormatOptions` to control the output format of complex types. 3. Modify some of result files of test cases.
apache · Jul 17, 2024 · 8bbba2a · 8bbba2a
1 parent ee8fbd7
commit 8bbba2a
Show file tree

Hide file tree

Showing 51 changed files with 1,115 additions and 722 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.ArrayType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
 
@@ -132,21 +133,21 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
+    public String getStringValueForArray(FormatOptions options) {
         List<String> list = new ArrayList<>(children.size());
-        children.forEach(v -> list.add(v.getStringValueForArray()));
+        children.forEach(v -> list.add(v.getStringValueForArray(options)));
         return "[" + StringUtils.join(list, ", ") + "]";
     }
 
     @Override
-    public String getStringValueInFe() {
+    public String getStringValueInFe(FormatOptions options) {
         List<String> list = new ArrayList<>(children.size());
         children.forEach(v -> {
             String stringLiteral;
             if (v instanceof NullLiteral) {
-                stringLiteral = "null";
+                stringLiteral = options.getNullFormat();
             } else {
-                stringLiteral = getStringLiteralForComplexType(v);
+                stringLiteral = getStringLiteralForComplexType(v, options);
             }
             // we should use type to decide we output array is suitable for json format
             list.add(stringLiteral);
@@ -155,14 +156,14 @@ public String getStringValueInFe() {
     }
 
     @Override
-    public String getStringValueForStreamLoad() {
+    public String getStringValueForStreamLoad(FormatOptions options) {
         List<String> list = new ArrayList<>(children.size());
         children.forEach(v -> {
             String stringLiteral;
             if (v instanceof NullLiteral) {
                 stringLiteral = "null";
             } else {
-                stringLiteral = getStringLiteralForStreamLoad(v);
+                stringLiteral = getStringLiteralForStreamLoad(v, options);
             }
             // we should use type to decide we output array is suitable for json format
             list.add(stringLiteral);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java
@@ -23,6 +23,7 @@
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.thrift.TBoolLiteral;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
@@ -111,8 +112,8 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -30,6 +30,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.catalog.TypeUtils;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.Pair;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.thrift.TExpr;
@@ -571,8 +572,8 @@ public boolean isNullable() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return children.get(0).getStringValueForArray();
+    public String getStringValueForArray(FormatOptions options) {
+        return children.get(0).getStringValueForArray(options);
     }
 
     public void setNotFold(boolean notFold) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -24,6 +24,7 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.InvalidFormatException;
 import org.apache.doris.nereids.util.DateUtils;
 import org.apache.doris.qe.SessionVariable;
@@ -730,8 +731,8 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 
     public void roundCeiling(int newScale) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java
@@ -22,6 +22,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.NotImplementedException;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.qe.SessionVariable;
@@ -262,7 +263,7 @@ public int compareLiteral(LiteralExpr expr) {
     }
 
     @Override
-    public String getStringValueInFe() {
+    public String getStringValueInFe(FormatOptions options) {
         return value.toPlainString();
     }
 
@@ -277,8 +278,8 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -39,6 +39,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.TreeNode;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.nereids.util.Utils;
@@ -2173,11 +2174,11 @@ public String getStringValue() {
         return "";
     }
 
-    public String getStringValueInFe() {
+    public String getStringValueInFe(FormatOptions options) {
         return getStringValue();
     }
 
-    public String getStringValueForStreamLoad() {
+    public String getStringValueForStreamLoad(FormatOptions options) {
         return getStringValue();
     }
 
@@ -2186,7 +2187,7 @@ public String getStringValueForStreamLoad() {
     // ["1", "2", "3"]
     // ["a", "b", "c"]
     // [["1", "2", "3"], ["1"], ["3"]]
-    public String getStringValueForArray() {
+    public String getStringValueForArray(FormatOptions options) {
         return null;
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
@@ -22,6 +22,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.NotImplementedException;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
@@ -151,7 +152,7 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueInFe() {
+    public String getStringValueInFe(FormatOptions options) {
         if (type == Type.TIME || type == Type.TIMEV2) {
             // FloatLiteral used to represent TIME type, here we need to remove apostrophe from timeStr
             // for example '11:22:33' -> 11:22:33
@@ -163,13 +164,13 @@ public String getStringValueInFe() {
     }
 
     @Override
-    public String getStringValueForArray() {
+    public String getStringValueForArray(FormatOptions options) {
         String ret = getStringValue();
         if (type == Type.TIME || type == Type.TIMEV2) {
             // here already wrapped in ''
             ret = ret.substring(1, ret.length() - 1);
         }
-        return "\"" + ret + "\"";
+        return options.getNestedStringWrapper() + ret + options.getNestedStringWrapper();
     }
 
     public static Type getDefaultTimeType(Type type) throws AnalysisException {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
 import org.apache.doris.thrift.TIPv4Literal;
@@ -139,7 +140,7 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
 import org.apache.doris.thrift.TIPv6Literal;
@@ -104,7 +105,7 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.NotImplementedException;
 import org.apache.doris.common.util.ByteBufferUtil;
 import org.apache.doris.qe.ConnectContext;
@@ -288,8 +289,8 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
@@ -100,7 +101,7 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
+    public String getStringValueForArray(FormatOptions options) {
         return null;
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
@@ -197,8 +198,8 @@ public String getStringValue() {
     }
 
     @Override
-    public String getStringValueForArray() {
-        return "\"" + getStringValue() + "\"";
+    public String getStringValueForArray(FormatOptions options) {
+        return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
@@ -25,6 +25,7 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.NotImplementedException;
 import org.apache.doris.mysql.MysqlProto;
 import org.apache.doris.thrift.TExprNode;
@@ -113,30 +114,29 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti
         return literalExpr;
     }
 
-
-    public static String getStringLiteralForComplexType(Expr v) {
+    public static String getStringLiteralForComplexType(Expr v, FormatOptions options) {
         if (!(v instanceof NullLiteral) && v.getType().isScalarType()
                 && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
                 || v.getType() == Type.BOOLEAN)) {
-            return v.getStringValueInFe();
+            return v.getStringValueInFe(options);
         } else if (v.getType().isComplexType()) {
             // these type should also call getStringValueInFe which should handle special case for itself
-            return v.getStringValueInFe();
+            return v.getStringValueInFe(options);
         } else {
-            return v.getStringValueForArray();
+            return v.getStringValueForArray(options);
         }
     }
 
-    public static String getStringLiteralForStreamLoad(Expr v) {
+    public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options) {
         if (!(v instanceof NullLiteral) && v.getType().isScalarType()
                 && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
                 || v.getType() == Type.BOOLEAN)) {
-            return v.getStringValueInFe();
+            return v.getStringValueInFe(options);
         } else if (v.getType().isComplexType()) {
             // these type should also call getStringValueInFe which should handle special case for itself
-            return v.getStringValueForStreamLoad();
+            return v.getStringValueForStreamLoad(options);
         } else {
-            return v.getStringValueForArray();
+            return v.getStringValueForArray(options);
         }
     }
 
@@ -265,12 +265,12 @@ public int compareTo(LiteralExpr literalExpr) {
     @Override
     public abstract String getStringValue();
 
-    public String getStringValueInFe() {
+    public String getStringValueInFe(FormatOptions options) {
         return getStringValue();
     }
 
     @Override
-    public abstract String getStringValueForArray();
+    public abstract String getStringValueForArray(FormatOptions options);
 
     public long getLongValue() {
         return 0;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.MapType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FormatOptions;
 import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
@@ -170,25 +171,27 @@ private String getStringValue(Expr expr) {
     }
 
     @Override
-    public String getStringValueForArray() {
+    public String getStringValueForArray(FormatOptions options) {
         List<String> list = new ArrayList<>(children.size());
         for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
-            list.add(children.get(i).getStringValueForArray() + ":" + children.get(i + 1).getStringValueForArray());
+            list.add(children.get(i).getStringValueForArray(options)
+                    + options.getMapKeyDelim()
+                    + children.get(i + 1).getStringValueForArray(options));
         }
         return "{" + StringUtils.join(list, ", ") + "}";
     }
 
     @Override
-    public String getStringValueInFe() {
+    public String getStringValueInFe(FormatOptions options) {
         List<String> list = new ArrayList<>(children.size());
         for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
             // we should use type to decide we output array is suitable for json format
             if (children.get(i).getType().isComplexType()) {
                 // map key type do not support complex type
-                throw new UnsupportedOperationException("Unsupport key type for MAP: " + children.get(i).getType());
+                throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType());
             }
-            list.add(getStringLiteralForComplexType(children.get(i))
-                    + ":" + getStringLiteralForComplexType(children.get(i + 1)));
+            list.add(getStringLiteralForComplexType(children.get(i), options)
+                    + options.getMapKeyDelim() + getStringLiteralForComplexType(children.get(i + 1), options));
         }
         return "{" + StringUtils.join(list, ", ") + "}";
     }