Merge remote-tracking branch 'upstream/dev' into dev

docs/en/concept/schema-feature.md

@@ -91,7 +91,9 @@ columns = [
 
 #### How to declare type supported
 
-SeaTunnel provides a simple and direct way to declare basic types. The keyword names for basic types can be used directly as type declarations, and SeaTunnel is case-insensitive to type keywords. Basic type keywords include `string`, `boolean`, `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `date`, `time`, `timestamp`, and `null`. For example, if you need to declare a field with integer type, you can simply define the field as `int` or `"int"`.
+SeaTunnel provides a simple and direct way to declare basic types. Basic type keywords include `string`, `boolean`, `tinyint`, `smallint`, `int`, `bigint`, `float`, `double`, `date`, `time`, `timestamp`, and `null`. The keyword names for basic types can be used directly as type declarations, and SeaTunnel is case-insensitive to type keywords. For example, if you need to declare a field with integer type, you can simply define the field as `int` or `"int"`.
+
+> The null type declaration must be enclosed in double quotes, like `"null"`. This approach helps avoid confusion with [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md)'s `null` type which represents undefined object.
 
 When declaring complex types (such as **decimal**, **array**, **map**, and **row**), pay attention to specific considerations.
 - When declaring a decimal type, precision and scale settings are required, and the type definition follows the format `decimal(precision, scale)`. It's essential to emphasize that the declaration of the decimal type must be enclosed in `"`; you cannot use the type name directly, as with basic types. For example, when declaring a decimal field with precision 10 and scale 2, you specify the field type as `"decimal(10,2)"`.

docs/en/connector-v2/source/FakeSource.md

@@ -68,12 +68,13 @@ just for some test cases such as type conversion or connector new feature testin
 
 ### Simple:
 
-> This example Randomly generates data of a specified type
+> This example Randomly generates data of a specified type. If you want to learn how to declare field types, click [here](../../concept/schema-feature.md#how-to-declare-type-supported).
 
 ```hocon
 schema = {
   fields {
     c_map = "map<string, array<int>>"
+    c_map_nest = "map<string, {c_int = int, c_string = string}>"
     c_array = "array<int>"
     c_string = string
     c_boolean = boolean
@@ -190,6 +191,8 @@ source {
 }
 ```
 
+> Due to the constraints of the [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md) specification, users cannot directly create byte sequence objects. FakeSource uses strings to assign `bytes` type values. In the example above, the `bytes` type field is assigned `"bWlJWmo="`, which is encoded from "miIZj" with **base64**. Hence, when assigning values to `bytes` type fields, please use strings encoded with **base64**.
+
 ### Specified Data number Simple:
 
 > This case specifies the number of data generated and the length of the generated value

release-note.md

@@ -187,6 +187,7 @@
 - [Transform-V2] Add support CatalogTable for FilterFieldTransform (#4422)
 - [Transform-V2] Add catalog support for SQL Transform plugin (#4819)
 - [Connector-V2] [Assert] Support check the precision and scale of Decimal type (#6110)
+- [Connector-V2] [Assert] Support field type assert and field value equality assert for full data types (#6275)
 
 ### Zeta(ST-Engine)
 

seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/SeaTunnelDataTypeConvertorUtil.java

@@ -43,7 +43,8 @@ public static SeaTunnelDataType<?> deserializeSeaTunnelDataType(
             String field, String columnType) {
         SqlType sqlType = null;
         try {
-            sqlType = SqlType.valueOf(columnType.toUpperCase().replace(" ", ""));
+            String compatible = compatibleTypeDeclare(columnType);
+            sqlType = SqlType.valueOf(compatible.toUpperCase().replace(" ", ""));
         } catch (IllegalArgumentException e) {
             // nothing
         }
@@ -84,6 +85,32 @@ public static SeaTunnelDataType<?> deserializeSeaTunnelDataType(
         }
     }
 
+    /**
+     * User-facing data type declarations will adhere to the specifications outlined in
+     * schema-feature.md. To maintain backward compatibility, this function will transform type
+     * declarations into standard form, including: <code>long -> bigint</code>, <code>
+     * short -> smallint</code>, and <code>byte -> tinyint</code>.
+     *
+     * <p>In a future version, user-facing data type declarations will strictly follow the
+     * specifications, and this function will be removed.
+     *
+     * @param declare
+     * @return compatible type
+     */
+    @Deprecated
+    private static String compatibleTypeDeclare(String declare) {
+        switch (declare.trim().toUpperCase()) {
+            case "LONG":
+                return "BIGINT";
+            case "SHORT":
+                return "SMALLINT";
+            case "BYTE":
+                return "TINYINT";
+            default:
+                return declare;
+        }
+    }
+
     private static SeaTunnelDataType<?> parseComplexDataType(String field, String columnStr) {
         String column = columnStr.toUpperCase().replace(" ", "");
         if (column.startsWith(SqlType.MAP.name())) {

seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/SeaTunnelDataTypeConvertorUtilTest.java

@@ -17,6 +17,11 @@
 
 package org.apache.seatunnel.api.table.catalog;
 
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.MapType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
 import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException;
 
 import org.junit.jupiter.api.Assertions;
@@ -89,4 +94,76 @@ void testParseWithUnsupportedType() {
                         "Unsupported parse SeaTunnel Type from '%s'.", invalidTypeDeclaration);
         Assertions.assertEquals(expectedMsg6, exception6.getMessage());
     }
+
+    @Test
+    public void testCompatibleTypeDeclare() {
+        SeaTunnelDataType<?> longType =
+                SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType("c_long", "long");
+        Assertions.assertEquals(BasicType.LONG_TYPE, longType);
+
+        SeaTunnelDataType<?> shortType =
+                SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType("c_short", "short");
+        Assertions.assertEquals(BasicType.SHORT_TYPE, shortType);
+
+        SeaTunnelDataType<?> byteType =
+                SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType("c_byte", "byte");
+        Assertions.assertEquals(BasicType.BYTE_TYPE, byteType);
+
+        ArrayType<?, ?> longArrayType =
+                (ArrayType<?, ?>)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_long_array", "array<long>");
+        Assertions.assertEquals(ArrayType.LONG_ARRAY_TYPE, longArrayType);
+
+        ArrayType<?, ?> shortArrayType =
+                (ArrayType<?, ?>)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_short_array", "array<short>");
+        Assertions.assertEquals(ArrayType.SHORT_ARRAY_TYPE, shortArrayType);
+
+        ArrayType<?, ?> byteArrayType =
+                (ArrayType<?, ?>)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_byte_array", "array<byte>");
+        Assertions.assertEquals(ArrayType.BYTE_ARRAY_TYPE, byteArrayType);
+
+        MapType<?, ?> longMapType =
+                (MapType<?, ?>)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_long_map", "map<long, long>");
+        Assertions.assertEquals(BasicType.LONG_TYPE, longMapType.getKeyType());
+        Assertions.assertEquals(BasicType.LONG_TYPE, longMapType.getValueType());
+
+        MapType<?, ?> shortMapType =
+                (MapType<?, ?>)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_short_map", "map<short, short>");
+        Assertions.assertEquals(BasicType.SHORT_TYPE, shortMapType.getKeyType());
+        Assertions.assertEquals(BasicType.SHORT_TYPE, shortMapType.getValueType());
+
+        MapType<?, ?> byteMapType =
+                (MapType<?, ?>)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_byte_map", "map<byte, byte>");
+        Assertions.assertEquals(BasicType.BYTE_TYPE, byteMapType.getKeyType());
+        Assertions.assertEquals(BasicType.BYTE_TYPE, byteMapType.getValueType());
+
+        SeaTunnelRowType longRow =
+                (SeaTunnelRowType)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_long_row", "{c = long}");
+        Assertions.assertEquals(BasicType.LONG_TYPE, longRow.getFieldType(0));
+
+        SeaTunnelRowType shortRow =
+                (SeaTunnelRowType)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_short_row", "{c = short}");
+        Assertions.assertEquals(BasicType.SHORT_TYPE, shortRow.getFieldType(0));
+
+        SeaTunnelRowType byteRow =
+                (SeaTunnelRowType)
+                        SeaTunnelDataTypeConvertorUtil.deserializeSeaTunnelDataType(
+                                "c_byte_row", "{c = byte}");
+        Assertions.assertEquals(BasicType.BYTE_TYPE, byteRow.getFieldType(0));
+    }
 }

seatunnel-connectors-v2/connector-assert/pom.xml

@@ -33,6 +33,11 @@
             <artifactId>connector-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-format-json</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 
 </project>