Support new data type: DATE, TIMESTAMP, BLOB and STRING

apache · May 28, 2024 · cb1bca6 · cb1bca6
1 parent 231910e
commit cb1bca6
Show file tree

Hide file tree

Showing 144 changed files with 2,896 additions and 534 deletions.
diff --git a/example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java b/example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java
@@ -30,6 +30,7 @@
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.time.LocalDate;
 
 public class JDBCExample {
   private static final Logger LOGGER = LoggerFactory.getLogger(JDBCExample.class);
@@ -51,9 +52,17 @@ public static void main(String[] args) throws ClassNotFoundException, SQLExcepti
           "CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=INT64, ENCODING=RLE, COMPRESSOR=SNAPPY");
       statement.execute(
           "CREATE TIMESERIES root.sg1.d1.s3 WITH DATATYPE=INT64, ENCODING=RLE, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s4 WITH DATATYPE=DATE, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s5 WITH DATATYPE=TIMESTAMP, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s6 WITH DATATYPE=BLOB, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s7 WITH DATATYPE=STRING, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
 
       for (int i = 0; i <= 100; i++) {
-        statement.addBatch(prepareInsertStatment(i));
+        statement.addBatch(prepareInsertStatement(i));
       }
       statement.executeBatch();
       statement.clearBatch();
@@ -96,15 +105,9 @@ private static void outputResult(ResultSet resultSet) throws SQLException {
     }
   }
 
-  private static String prepareInsertStatment(int time) {
-    return "insert into root.sg1.d1(timestamp, s1, s2, s3) values("
-        + time
-        + ","
-        + 1
-        + ","
-        + 1
-        + ","
-        + 1
-        + ")";
+  private static String prepareInsertStatement(int time) {
+    return String.format(
+        "insert into root.sg1.d1(timestamp, s1, s2, s3, s4, s5, s6, s7) values(%d, %d, %d, %d, \"%s\", %d, %s, \"%s\")",
+        time, 1, 1, 1, LocalDate.of(2024, 5, time % 31 + 1), time, "X'cafebabe'", time);
   }
 }
diff --git a/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java b/example/session/src/main/java/org/apache/iotdb/DataMigrationExample.java
@@ -155,6 +155,7 @@ public Void call() {
                 tablet.addValue(schemaList.get(j).getMeasurementId(), row, dataIter.getInt(j + 2));
                 break;
               case INT64:
+              case TIMESTAMP:
                 tablet.addValue(schemaList.get(j).getMeasurementId(), row, dataIter.getLong(j + 2));
                 break;
               case FLOAT:
@@ -166,9 +167,15 @@ public Void call() {
                     schemaList.get(j).getMeasurementId(), row, dataIter.getDouble(j + 2));
                 break;
               case TEXT:
+              case STRING:
                 tablet.addValue(
                     schemaList.get(j).getMeasurementId(), row, dataIter.getString(j + 2));
                 break;
+              case DATE:
+              case BLOB:
+                tablet.addValue(
+                    schemaList.get(j).getMeasurementId(), row, dataIter.getObject(j + 2));
+                break;
               default:
                 LOGGER.info("Migration of this type of data is not supported");
             }

diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
@@ -199,6 +199,7 @@ public class ClusterConstant {
   public static final int NODE_START_TIMEOUT = 100;
   public static final int PROBE_TIMEOUT_MS = 2000;
   public static final int NODE_NETWORK_TIMEOUT_MS = 0;
+  public static final String ZERO_TIME_ZONE = "GMT+0";
 
   public static final String DELIMITER = ",";
   public static final String TAB = "  ";

diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java
@@ -479,7 +479,9 @@ protected NodeConnection getWriteConnectionWithSpecifiedDataNode(
     String endpoint = dataNode.getIp() + ":" + dataNode.getPort();
     Connection writeConnection =
         DriverManager.getConnection(
-            Config.IOTDB_URL_PREFIX + endpoint + getParam(version, NODE_NETWORK_TIMEOUT_MS),
+            Config.IOTDB_URL_PREFIX
+                + endpoint
+                + getParam(version, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE),
             System.getProperty("User", username),
             System.getProperty("Password", password));
     return new NodeConnection(
@@ -521,7 +523,9 @@ protected List<NodeConnection> getReadConnections(
           () -> {
             Connection readConnection =
                 DriverManager.getConnection(
-                    Config.IOTDB_URL_PREFIX + endpoint + getParam(version, NODE_NETWORK_TIMEOUT_MS),
+                    Config.IOTDB_URL_PREFIX
+                        + endpoint
+                        + getParam(version, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE),
                     System.getProperty("User", username),
                     System.getProperty("Password", password));
             return new NodeConnection(
@@ -559,7 +563,7 @@ protected void testJDBCConnection() {
                       DriverManager.getConnection(
                           Config.IOTDB_URL_PREFIX
                               + dataNodeEndpoint
-                              + getParam(null, NODE_NETWORK_TIMEOUT_MS),
+                              + getParam(null, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE),
                           System.getProperty("User", "root"),
                           System.getProperty("Password", "root"))) {
                 logger.info("Successfully connecting to DataNode: {}.", dataNodeEndpoint);
@@ -584,12 +588,15 @@ protected void testJDBCConnection() {
     }
   }
 
-  private String getParam(Constant.Version version, int timeout) {
+  private String getParam(Constant.Version version, int timeout, String timeZone) {
     StringBuilder sb = new StringBuilder("?");
     sb.append(Config.NETWORK_TIMEOUT).append("=").append(timeout);
     if (version != null) {
       sb.append("&").append(VERSION).append("=").append(version);
     }
+    if (timeZone != null) {
+      sb.append("&").append(Config.TIME_ZONE).append("=").append(timeZone);
+    }
     return sb.toString();
   }
 

diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java
@@ -77,6 +77,10 @@ private static void createTimeSeries() {
       statement.execute("CREATE TIMESERIES root.vehicle.d1.s1 with datatype=INT32,encoding=PLAIN");
       statement.execute("CREATE TIMESERIES root.vehicle.d1.s2 with datatype=INT32,encoding=PLAIN");
       statement.execute("CREATE TIMESERIES root.vehicle.d1.s3 with datatype=TEXT,encoding=PLAIN");
+      statement.execute("CREATE TIMESERIES root.vehicle.d1.s4 with datatype=STRING,encoding=PLAIN");
+      statement.execute("CREATE TIMESERIES root.vehicle.d1.s5 with datatype=DATE,encoding=PLAIN");
+      statement.execute(
+          "CREATE TIMESERIES root.vehicle.d1.s6 with datatype=TIMESTAMP,encoding=PLAIN");
       statement.execute("CREATE TIMESERIES root.vehicle.d2.s1 with datatype=FLOAT,encoding=PLAIN");
       statement.execute("CREATE TIMESERIES root.vehicle.d2.s2 with datatype=DOUBLE,encoding=PLAIN");
       statement.execute(
@@ -92,11 +96,15 @@ private static void generateData() {
       for (int i = 1; i <= ITERATION_TIMES; ++i) {
         statement.execute(
             String.format(
-                "insert into root.vehicle.d1(timestamp,s1,s2,s3) values(%d,%d,%d,%s)", i, i, i, i));
+                "insert into root.vehicle.d1(timestamp,s1,s2,s3,s4,s6) values(%d,%d,%d,%s,%s,%d)",
+                i, i, i, i, i, i));
         statement.execute(
             (String.format(
                 "insert into root.vehicle.d2(timestamp,s1,s2) values(%d,%d,%d)", i, i, i)));
       }
+      statement.execute("insert into root.vehicle.d1(timestamp,s5) values(1,'2024-01-01')");
+      statement.execute("insert into root.vehicle.d1(timestamp,s5) values(2,'2024-01-02')");
+      statement.execute("insert into root.vehicle.d1(timestamp,s5) values(3,'2024-01-03')");
     } catch (SQLException throwable) {
       fail(throwable.getMessage());
     }
@@ -624,6 +632,36 @@ public void testRegularLikeInExpressions() {
         }
         Assert.assertFalse(rs.next());
       }
+
+      String query2 =
+          "SELECT s1 FROM root.vehicle.d1 WHERE s4 LIKE '_' && s4 REGEXP '[0-9]' && s4 IN ('4', '2', '3')";
+      try (ResultSet rs = statement.executeQuery(query2)) {
+        for (int i = 2; i <= 4; i++) {
+          Assert.assertTrue(rs.next());
+          Assert.assertEquals(i, rs.getLong(1));
+        }
+        Assert.assertFalse(rs.next());
+      }
+
+      String query3 =
+          "SELECT s1 FROM root.vehicle.d1 WHERE s5 IN ('2024-01-01', '2024-01-02', '2024-01-03')";
+      try (ResultSet rs = statement.executeQuery(query3)) {
+        for (int i = 1; i <= 3; i++) {
+          Assert.assertTrue(rs.next());
+          Assert.assertEquals(i, rs.getLong(1));
+        }
+        Assert.assertFalse(rs.next());
+      }
+
+      String query4 = "SELECT s1 FROM root.vehicle.d1 WHERE s6 IN (1, 2, 3)";
+      try (ResultSet rs = statement.executeQuery(query4)) {
+        for (int i = 1; i <= 3; i++) {
+          Assert.assertTrue(rs.next());
+          Assert.assertEquals(i, rs.getLong(1));
+        }
+        Assert.assertFalse(rs.next());
+      }
+
     } catch (SQLException e) {
       e.printStackTrace();
       Assert.fail(e.getMessage());

diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java
@@ -25,6 +25,7 @@
 import org.apache.iotdb.itbase.category.LocalStandaloneIT;
 import org.apache.iotdb.rpc.TSStatusCode;
 
+import org.apache.tsfile.enums.TSDataType;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -38,12 +39,17 @@
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.sql.Timestamp;
 import java.sql.Types;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -1125,4 +1131,62 @@ public void testFromFuzzyMatching() {
       fail();
     }
   }
+
+  @Test
+  public void testNewDataType() {
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+
+      statement.execute("CREATE DATABASE root.sg1");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s4 WITH DATATYPE=DATE, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s5 WITH DATATYPE=TIMESTAMP, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s6 WITH DATATYPE=BLOB, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      statement.execute(
+          "CREATE TIMESERIES root.sg1.d1.s7 WITH DATATYPE=STRING, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+      for (int i = 1; i <= 10; i++) {
+        statement.execute(
+            String.format(
+                "insert into root.sg1.d1(timestamp, s4, s5, s6, s7) values(%d, \"%s\", %d, %s, \"%s\")",
+                i, LocalDate.of(2024, 5, i % 31 + 1), i, "X'cafebabe'", i));
+      }
+
+      try (ResultSet resultSet = statement.executeQuery("select * from root.**")) {
+        final ResultSetMetaData metaData = resultSet.getMetaData();
+        final int columnCount = metaData.getColumnCount();
+        assertEquals(5, columnCount);
+        HashMap<Integer, TSDataType> columnType = new HashMap<>();
+        for (int i = 1; i < columnCount; i++) {
+          if (metaData.getColumnLabel(i).endsWith("s4")) {
+            columnType.put(i, TSDataType.DATE);
+          } else if (metaData.getColumnLabel(i).endsWith("s5")) {
+            columnType.put(i, TSDataType.TIMESTAMP);
+          } else if (metaData.getColumnLabel(i).endsWith("s6")) {
+            columnType.put(i, TSDataType.BLOB);
+          } else if (metaData.getColumnLabel(i).endsWith("s7")) {
+            columnType.put(i, TSDataType.TEXT);
+          }
+        }
+        byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE};
+        while (resultSet.next()) {
+          long time = resultSet.getLong(1);
+          Date date = resultSet.getDate(2);
+          Timestamp timestamp = resultSet.getTimestamp(3);
+          byte[] blob = resultSet.getBytes(4);
+          String text = resultSet.getString(5);
+          assertEquals(2024 - 1900, date.getYear());
+          assertEquals(5 - 1, date.getMonth());
+          assertEquals(time % 31 + 1, date.getDate());
+          assertEquals(time, timestamp.getTime());
+          assertArrayEquals(byteArray, blob);
+          assertEquals(String.valueOf(time), text);
+        }
+      }
+
+    } catch (SQLException e) {
+      fail();
+    }
+  }
 }
diff --git a/...ation-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java b/...ation-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java
@@ -179,7 +179,7 @@ public void testStringLiteralIllegalCase() {
     String errorMsg =
         TSStatusCode.SQL_PARSE_ERROR.getStatusCode()
             + ": Error occurred while parsing SQL to physical plan: "
-            + "line 1:45 mismatched input 'string' expecting {FALSE, NAN, NOW, NULL, TRUE, '-', '+', '/', '.', STRING_LITERAL, DATETIME_LITERAL, INTEGER_LITERAL, EXPONENT_NUM_PART}";
+            + "line 1:45 mismatched input 'string' expecting {FALSE, NAN, NOW, NULL, TRUE, '-', '+', '/', '.', STRING_LITERAL, BINARY_LITERAL, DATETIME_LITERAL, INTEGER_LITERAL, EXPONENT_NUM_PART}";
     try (Connection connection = EnvFactory.getEnv().getConnection();
         Statement statement = connection.createStatement()) {
       statement.execute("CREATE TIMESERIES root.sg1.d1.s1 TEXT");
@@ -198,7 +198,7 @@ public void testStringLiteralIllegalCase() {
     String errorMsg1 =
         TSStatusCode.SQL_PARSE_ERROR.getStatusCode()
             + ": Error occurred while parsing SQL to physical plan: "
-            + "line 1:45 mismatched input '`string`' expecting {FALSE, NAN, NOW, NULL, TRUE, '-', '+', '/', '.', STRING_LITERAL, DATETIME_LITERAL, INTEGER_LITERAL, EXPONENT_NUM_PART}";
+            + "line 1:45 mismatched input '`string`' expecting {FALSE, NAN, NOW, NULL, TRUE, '-', '+', '/', '.', STRING_LITERAL, BINARY_LITERAL, DATETIME_LITERAL, INTEGER_LITERAL, EXPONENT_NUM_PART}";
     try (Connection connection = EnvFactory.getEnv().getConnection();
         Statement statement = connection.createStatement()) {
       // wrap STRING_LITERAL with ``