diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 1760301ac737..9126e9cb1e8b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -97,6 +97,24 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptFlag(boolean encryptFlag) { + setProperty("encrypt_flag", String.valueOf(encryptFlag)); + return this; + } + + @Override + public CommonConfig setEncryptType(String encryptType) { + setProperty("encrypt_type", encryptType); + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + setProperty("encrypt_key_path", encryptKeyPath); + return this; + } + @Override public CommonConfig setUdfMemoryBudgetInMB(float udfCollectorMemoryBudgetInMB) { // udf_memory_budget_in_mb diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 969d4bb8d41c..b5598de6f693 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -75,6 +75,27 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptFlag(boolean encryptFlag) { + cnConfig.setProperty("encrypt_flag", String.valueOf(encryptFlag)); + dnConfig.setProperty("encrypt_flag", String.valueOf(encryptFlag)); + return this; + } + + @Override + public CommonConfig setEncryptType(String encryptType) { + cnConfig.setProperty("encrypt_type", encryptType); + dnConfig.setProperty("encrypt_type", encryptType); + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + cnConfig.setProperty("encrypt_key_path", encryptKeyPath); + dnConfig.setProperty("encrypt_key_path", encryptKeyPath); + return this; + } + @Override public CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs) { cnConfig.setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(maxMs); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index cee8aadf05f9..787b16139169 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -54,6 +54,21 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptFlag(boolean encryptFlag) { + return this; + } + + @Override + public CommonConfig setEncryptType(String encryptType) { + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + return this; + } + @Override public CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index d806be8db1dc..5dffdfe22e59 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -36,6 +36,12 @@ public interface CommonConfig { CommonConfig setCompressor(String compressor); + CommonConfig setEncryptFlag(boolean encryptFlag); + + CommonConfig setEncryptType(String encryptType); + + CommonConfig setEncryptKeyPath(String encryptKeyPath); + CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs); CommonConfig setUdfMemoryBudgetInMB(float udfCollectorMemoryBudgetInMB); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java new file mode 100644 index 000000000000..b63e409a3e31 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBEncryptionValueQueryIT { + private static String[] sqls = + new String[] { + "CREATE DATABASE root.ln", + "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465600000,true)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465660000,true)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465720000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465780000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465840000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465900000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465960000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466020000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466080000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466140000,false)", + "create timeseries root.ln.wf01.wt01.temperature with datatype=FLOAT,encoding=RLE", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600000,25.957603)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465660000,24.359503)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465720000,20.092794)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465780000,20.182663)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465840000,21.125198)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465900000,22.720892)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465960000,20.71);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466020000,21.451046);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466080000,22.57987);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466140000,20.98177);", + "create timeseries root.ln.wf02.wt02.hardware with datatype=TEXT,encoding=PLAIN", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465600000,'v2')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465660000,'v2')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465720000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465780000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465840000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465900000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465960000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466020000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466080000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466140000,'v1')", + "create timeseries root.ln.wf02.wt02.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465600000,true)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465660000,true)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465720000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465780000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465840000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465900000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465960000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466020000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466080000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466140000,false)", + "flush", + "CREATE DATABASE root.sgcc", + "create timeseries root.sgcc.wf03.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465600000,true)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465660000,true)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465720000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465780000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465840000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465900000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465960000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466020000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466080000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466140000,false)", + "create timeseries root.sgcc.wf03.wt01.temperature with datatype=FLOAT,encoding=RLE", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465600000,25.957603)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465660000,24.359503)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465720000,20.092794)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465780000,20.182663)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465840000,21.125198)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465900000,22.720892)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465960000,20.71)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466020000,21.451046)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466080000,22.57987)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466140000,20.98177)", + "flush", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEncryptFlag(true) + .setEncryptType("org.apache.tsfile.encrypt.AES128"); + EnvFactory.getEnv().initClusterEnvironment(); + importData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void importData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectTest() { + String[] retArray = + new String[] { + "1509465600000,true,25.96,v2,true,true,25.96,", + "1509465660000,true,24.36,v2,true,true,24.36,", + "1509465720000,false,20.09,v1,false,false,20.09,", + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + "1509466080000,false,22.58,v1,false,false,22.58,", + "1509466140000,false,20.98,v1,false,false,20.98,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = statement.executeQuery("select * from root.** where time>10"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void LimitTest() { + String[] retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // test 1: fetchSize < limitNumber + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + + ResultSet resultSet = + statement.executeQuery("select * from root.** where time>10 limit 5 offset 3"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + // test 1: fetchSize > limitNumber + statement.setFetchSize(10000); + Assert.assertEquals(10000, statement.getFetchSize()); + resultSet = statement.executeQuery("select * from root.** where time>10 limit 5 offset 3"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void InTest() { + String[] retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // test 1: fetchSize < limitNumber + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + ResultSet resultSet = + statement.executeQuery( + "select * from root.** where time in (1509465780000, 1509465840000, 1509465900000, 1509465960000, 1509466020000)"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + retArray = + new String[] { + "1509465600000,true,25.96,v2,true,true,25.96,", + "1509465660000,true,24.36,v2,true,true,24.36,", + "1509465720000,false,20.09,v1,false,false,20.09,", + "1509466080000,false,22.58,v1,false,false,22.58,", + "1509466140000,false,20.98,v1,false,false,20.98,", + }; + resultSet = + statement.executeQuery( + "select * from root.** where time not in (1509465780000, 1509465840000, 1509465900000, 1509465960000, 1509466020000)"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466080000,false,22.58,v1,false,false,22.58,", + }; + + resultSet = + statement.executeQuery( + "select * from root.** where root.ln.wf01.wt01.temperature in (20.18, 20.71, 22.58)"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } + + @Test + public void testRightTextQuery() { + // Text type uses the equal operator to query the correct result + String[] retArray = + new String[] { + "1509465600000,v2,", "1509465660000,v2,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + ResultSet resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf02.wt02.hardware,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + resultSetMetaData = resultSet.getMetaData(); + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + } catch (Exception e) { + Assert.assertNull(e.getMessage()); + } + } + + @Test + public void RegexpTest() { + String[] retArray = + new String[] { + "1509465600000,v2,true,", "1509465660000,v2,true,", "1509465720000,v1,false,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + // Matches a string consisting of one lowercase letter and one digit. such as: "v1","v2" + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp '^[a-z][0-9]$' and time < 1509465780000"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + + retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp 'v*' "); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void RegexpNonExistTest() { + + // Match nonexistent string.'s.' is indicates that the starting with s and the last is any + // single character + String[] retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp 's.' "); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(0, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java new file mode 100644 index 000000000000..bbb2f173c36e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.it.env.EnvFactory; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IoTDBLoadEncryptedTsFileIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEncryptFlag(true) + .setEncryptType("org.apache.tsfile.encrypt.AES128"); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadSameWayEncryptedTsFileTest() { + String[] retArray = + new String[] { + "2,1,", "3,1,", "4,1,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg"); + statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateSameWayEncryptedFile(); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg.d1"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.testsg.d1.s1,", + new int[] { + Types.TIMESTAMP, Types.INTEGER, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void loadAnotherWayEncryptedTsFileTest() { + String unrecognizedType = "org.apache.tsfile.encrypt.SM4128"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg1"); + statement.execute("CREATE TIMESERIES root.testsg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateAnotherWayEncryptedFile(unrecognizedType); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg1.d1"); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("The encryption way of the TsFile is not supported.")); + } + } + + private File generateSameWayEncryptedFile() throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + config.setEncryptFlag("true"); + config.setEncryptType("org.apache.tsfile.encrypt.AES128"); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile, config)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema("s1", TSDataType.INT32), + EncryptUtils.getEncryptParameter(config)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + config.setEncryptFlag("false"); + config.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED"); + return tsfile; + } + + private File generateAnotherWayEncryptedFile(String unrecognizedType) throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + config.setEncryptFlag("true"); + config.setEncryptType("org.apache.tsfile.encrypt.AES128"); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile, config)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg1.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema("s1", TSDataType.INT32), + EncryptUtils.getEncryptParameter(config)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.setEncryptParam("2", unrecognizedType, EncryptUtils.getNormalKeyStr(config)); + writer.endFile(); + } + config.setEncryptFlag("false"); + config.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED"); + return tsfile; + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java new file mode 100644 index 000000000000..bdff3fb463c5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.it.env.EnvFactory; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IoTDBLoadPlainTsFileIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEncryptFlag(true) + .setEncryptType("org.apache.tsfile.encrypt.AES128"); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadNormalTsFileTest() { + String[] retArray = + new String[] { + "2,1,", "3,1,", "4,1,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg"); + statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateNormalFile(); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg.d1"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.testsg.d1.s1,", + new int[] { + Types.TIMESTAMP, Types.INTEGER, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("The encryption way of the TsFile is not supported.")); + } + } + + private File generateNormalFile() throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT32)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + return tsfile; + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 5a2be7740e07..7ecb0b527910 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1770,6 +1770,24 @@ private void loadTsFileProps(Properties properties) throws IOException { .setCompressor( properties.getProperty( "compressor", ConfigurationFileUtils.getConfigurationDefaultValue("compressor"))); + TSFileDescriptor.getInstance() + .getConfig() + .setEncryptFlag( + properties.getProperty( + "encrypt_flag", + ConfigurationFileUtils.getConfigurationDefaultValue("encrypt_flag"))); + TSFileDescriptor.getInstance() + .getConfig() + .setEncryptType( + properties.getProperty( + "encrypt_type", + ConfigurationFileUtils.getConfigurationDefaultValue("encrypt_type"))); + TSFileDescriptor.getInstance() + .getConfig() + .setEncryptKeyFromPath( + properties.getProperty( + "encrypt_key_path", + ConfigurationFileUtils.getConfigurationDefaultValue("encrypt_key_path"))); TSFileDescriptor.getInstance() .getConfig() .setMaxTsBlockSizeInBytes( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java index 41bef7fe3b00..28e8a1deab4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -47,6 +49,8 @@ public class AlignedSinglePageWholeChunkReader extends AbstractChunkReader { // chunk data of the time column private final ByteBuffer timeChunkDataBuffer; + private final EncryptParameter encryptParam; + // chunk headers of all the sub sensors private final List valueChunkHeaderList = new ArrayList<>(); // chunk data of all the sub sensors @@ -59,6 +63,7 @@ public AlignedSinglePageWholeChunkReader(Chunk timeChunk, List valueChunk super(Long.MIN_VALUE, null); this.timeChunkHeader = timeChunk.getHeader(); this.timeChunkDataBuffer = timeChunk.getData(); + this.encryptParam = timeChunk.getEncryptParam(); valueChunkList.forEach( chunk -> { @@ -120,8 +125,10 @@ private void skipCurrentPage(PageHeader timePageHeader, List valuePa private AlignedPageReader constructAlignedPageReader( PageHeader timePageHeader, List rawValuePageHeaderList) throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer timePageData = - ChunkReader.deserializePageData(timePageHeader, timeChunkDataBuffer, timeChunkHeader); + ChunkReader.deserializePageData( + timePageHeader, timeChunkDataBuffer, timeChunkHeader, decryptor); List valuePageHeaderList = new ArrayList<>(); List valuePageDataList = new ArrayList<>(); @@ -143,7 +150,7 @@ private AlignedPageReader constructAlignedPageReader( valuePageHeaderList.add(valuePageHeader); valuePageDataList.add( ChunkReader.deserializePageData( - valuePageHeader, valueChunkDataBufferList.get(i), valueChunkHeader)); + valuePageHeader, valueChunkDataBufferList.get(i), valueChunkHeader, decryptor)); valueDataTypeList.add(valueChunkHeader.getDataType()); valueDecoderList.add( Decoder.getDecoderByType( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java index cd85ae7866d5..ed831644988f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java @@ -21,8 +21,11 @@ import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.reader.chunk.AbstractChunkReader; @@ -32,16 +35,19 @@ import java.io.Serializable; import java.nio.ByteBuffer; +import static org.apache.tsfile.file.metadata.enums.CompressionType.UNCOMPRESSED; + public class SinglePageWholeChunkReader extends AbstractChunkReader { private final ChunkHeader chunkHeader; private final ByteBuffer chunkDataBuffer; + private final EncryptParameter encryptParam; public SinglePageWholeChunkReader(Chunk chunk) throws IOException { super(Long.MIN_VALUE, null); this.chunkHeader = chunk.getHeader(); this.chunkDataBuffer = chunk.getData(); - + this.encryptParam = chunk.getEncryptParam(); initAllPageReaders(); } @@ -56,9 +62,10 @@ private void initAllPageReaders() throws IOException { } private PageReader constructPageReader(PageHeader pageHeader) throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); return new PageReader( pageHeader, - deserializePageData(pageHeader, chunkDataBuffer, chunkHeader), + deserializePageData(pageHeader, chunkDataBuffer, chunkHeader, decryptor), chunkHeader.getDataType(), Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), defaultTimeDecoder, @@ -88,11 +95,14 @@ public static ByteBuffer readCompressedPageData(PageHeader pageHeader, ByteBuffe public static ByteBuffer uncompressPageData( PageHeader pageHeader, IUnCompressor unCompressor, ByteBuffer compressedPageData) throws IOException { + if (unCompressor.getCodecName() == UNCOMPRESSED) { + return compressedPageData; + } int compressedPageBodyLength = pageHeader.getCompressedSize(); - byte[] uncompressedPageData = new byte[pageHeader.getUncompressedSize()]; + ByteBuffer uncompressedPageData = ByteBuffer.allocate(pageHeader.getUncompressedSize()); try { unCompressor.uncompress( - compressedPageData.array(), 0, compressedPageBodyLength, uncompressedPageData, 0); + compressedPageData.array(), 0, compressedPageBodyLength, uncompressedPageData.array(), 0); } catch (Exception e) { throw new IOException( "Uncompress error! uncompress size: " @@ -101,16 +111,40 @@ public static ByteBuffer uncompressPageData( + pageHeader.getCompressedSize() + "page header: " + pageHeader - + e.getMessage()); + + e.getMessage(), + e); } - return ByteBuffer.wrap(uncompressedPageData); + return uncompressedPageData; + } + + public static ByteBuffer decrypt(IDecryptor decryptor, ByteBuffer buffer) { + if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return buffer; + } + return ByteBuffer.wrap( + decryptor.decrypt( + buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining())); + } + + public static ByteBuffer decryptAndUncompressPageData( + PageHeader pageHeader, + IUnCompressor unCompressor, + ByteBuffer compressedPageData, + IDecryptor decryptor) + throws IOException { + return uncompressPageData(pageHeader, unCompressor, decrypt(decryptor, compressedPageData)); } public static ByteBuffer deserializePageData( - PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader) throws IOException { + PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader, IDecryptor decryptor) + throws IOException { IUnCompressor unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()); ByteBuffer compressedPageBody = readCompressedPageData(pageHeader, chunkBuffer); - return uncompressPageData(pageHeader, unCompressor, compressedPageBody); + if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return uncompressPageData(pageHeader, unCompressor, compressedPageBody); + } else { + return decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageBody, decryptor); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java index a10e39bd6f29..67c5b787b8cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java @@ -44,6 +44,8 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.commons.io.FileUtils; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -53,6 +55,7 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -134,6 +137,13 @@ protected void analyzeSingleTsFile(final File tsFile) throw new SemanticException("Attempted to load a tree-model TsFile into table-model."); } + // check whether the encrypt type of the tsfile is supported + EncryptParameter param = reader.getEncryptParam(); + if (!Objects.equals(param.getType(), EncryptUtils.encryptParam.getType()) + || !Arrays.equals(param.getKey(), EncryptUtils.encryptParam.getKey())) { + throw new SemanticException("The encryption way of the TsFile is not supported."); + } + // construct tsfile resource final TsFileResource tsFileResource = constructTsFileResource(reader, tsFile); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java index f07cee07f63c..a75c9a87bbe5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java @@ -37,6 +37,8 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.commons.io.FileUtils; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -46,6 +48,7 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -118,6 +121,13 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx throw new LoadEmptyFileException(tsFile.getAbsolutePath()); } + // check whether the encrypt type of the tsfile is supported + EncryptParameter param = reader.getEncryptParam(); + if (!Objects.equals(param.getType(), EncryptUtils.encryptParam.getType()) + || !Arrays.equals(param.getKey(), EncryptUtils.encryptParam.getKey())) { + throw new SemanticException("The encryption way of the TsFile is not supported."); + } + // check whether the tsfile is tree-model or not // TODO: currently, loading a file with both tree-model and table-model data is not supported. // May need to support this and remove this check in the future. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java index 3ad81d399309..0a12a529abf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java @@ -119,7 +119,11 @@ public Chunk get( FileReaderManager.getInstance().get(chunkCacheKey.getFilePath(), true); Chunk chunk = reader.readMemChunk(chunkCacheKey.offsetOfChunkHeader); return new Chunk( - chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic); + chunk.getHeader(), + chunk.getData().duplicate(), + timeRangeList, + chunkStatistic, + chunk.getEncryptParam()); } Chunk chunk = lruCache.get(chunkCacheKey); @@ -129,7 +133,11 @@ public Chunk get( } return new Chunk( - chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic); + chunk.getHeader(), + chunk.getData().duplicate(), + timeRangeList, + chunkStatistic, + chunk.getEncryptParam()); } finally { SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( READ_CHUNK_ALL, System.nanoTime() - startTime); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java index 1ba5ff680ec0..7b1326f437f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java @@ -67,7 +67,7 @@ public void addTimeChunkToCache(String file, long offset, Chunk chunk) { chunk.getData(), chunk.getDeleteIntervalList(), chunk.getChunkStatistic(), - chunk.getDecryptor())); + chunk.getEncryptParam())); cachedTimeChunkSize += chunk.getHeader().getDataSize(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java index ff41e5e840e7..a94150deca19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java @@ -24,6 +24,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -39,7 +41,7 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; public class CompactionAlignedChunkReader { @@ -47,6 +49,8 @@ public class CompactionAlignedChunkReader { private final List valueChunkHeaderList = new ArrayList<>(); private final IUnCompressor timeUnCompressor; + + private final EncryptParameter encryptParam; private final boolean ignoreAllNullRows; private final Decoder timeDecoder = Decoder.getDecoderByType( @@ -65,6 +69,7 @@ public CompactionAlignedChunkReader( Chunk timeChunk, List valueChunkList, boolean ignoreAllNullRows) { ChunkHeader timeChunkHeader = timeChunk.getHeader(); this.timeUnCompressor = IUnCompressor.getUnCompressor(timeChunkHeader.getCompressionType()); + this.encryptParam = timeChunk.getEncryptParam(); this.timeDeleteIntervalList = timeChunk.getDeleteIntervalList(); this.valueDeleteIntervalList = new ArrayList<>(valueChunkList.size()); @@ -113,9 +118,11 @@ private IPointReader getPontReader( boolean ignoreAllNullRows) throws IOException { - // uncompress time page data + // decrypt and uncompress time page data + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer uncompressedTimePageData = - uncompressPageData(timePageHeader, timeUnCompressor, compressedTimePageData); + decryptAndUncompressPageData( + timePageHeader, timeUnCompressor, compressedTimePageData, decryptor); TimePageReader timePageReader = new TimePageReader(timePageHeader, uncompressedTimePageData, timeDecoder); timePageReader.setDeleteIntervalList(timeDeleteIntervalList); @@ -128,10 +135,11 @@ private IPointReader getPontReader( } else { ChunkHeader valueChunkHeader = valueChunkHeaderList.get(i); ByteBuffer uncompressedPageData = - uncompressPageData( + decryptAndUncompressPageData( valuePageHeaders.get(i), IUnCompressor.getUnCompressor(valueChunkHeader.getCompressionType()), - compressedValuePageDatas.get(i)); + compressedValuePageDatas.get(i), + decryptor); TSDataType valueType = valueChunkHeader.getDataType(); ValuePageReader valuePageReader = new ValuePageReader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java index 81836b43ea49..2b39064d36a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java @@ -22,6 +22,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -39,13 +41,15 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; public class CompactionChunkReader { private final ChunkHeader chunkHeader; private ByteBuffer chunkDataBuffer; private final IUnCompressor unCompressor; + + private final EncryptParameter encryptParam; private final Decoder timeDecoder = Decoder.getDecoderByType( TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), @@ -66,6 +70,7 @@ public CompactionChunkReader(Chunk chunk) { this.unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()); this.deleteIntervalList = chunk.getDeleteIntervalList(); this.chunkStatistic = chunk.getChunkStatistic(); + this.encryptParam = chunk.getEncryptParam(); } /** @@ -126,9 +131,10 @@ public static ByteBuffer readCompressedPageData(PageHeader pageHeader, ByteBuffe */ public TsBlock readPageData(PageHeader pageHeader, ByteBuffer compressedPageData) throws IOException { - // uncompress page data - ByteBuffer pageData = uncompressPageData(pageHeader, unCompressor, compressedPageData); - + // decrypt and uncompress page data + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); + ByteBuffer pageData = + decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageData, decryptor); // decode page data TSDataType dataType = chunkHeader.getDataType(); Decoder valueDecoder = Decoder.getDecoderByType(chunkHeader.getEncodingType(), dataType); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java index 41d88fc3d79b..5256204b5e4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java @@ -93,7 +93,8 @@ public List getPages() { chunk.getHeader().getDataType(), chunk.getHeader().getEncodingType(), chunkMetadata, - pageModifiedStatus)); + pageModifiedStatus, + chunk.getEncryptParam())); } return pageList; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java index 8da39abda78c..c01ed64e8b68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java @@ -22,23 +22,29 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; public class InstantPageLoader extends PageLoader { private ByteBuffer pageData; + private EncryptParameter encryptParam; + public InstantPageLoader() {} public InstantPageLoader( @@ -49,9 +55,11 @@ public InstantPageLoader( TSDataType dataType, TSEncoding encoding, ChunkMetadata chunkMetadata, - ModifiedStatus modifiedStatus) { + ModifiedStatus modifiedStatus, + EncryptParameter encryptParam) { super(file, pageHeader, compressionType, dataType, encoding, chunkMetadata, modifiedStatus); this.pageData = pageData; + this.encryptParam = encryptParam; } @Override @@ -62,7 +70,12 @@ public ByteBuffer getCompressedData() { @Override public ByteBuffer getUnCompressedData() throws IOException { IUnCompressor unCompressor = IUnCompressor.getUnCompressor(compressionType); - return uncompressPageData(pageHeader, unCompressor, pageData); + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); + if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return uncompressPageData(pageHeader, unCompressor, pageData); + } else { + return decryptAndUncompressPageData(pageHeader, unCompressor, pageData, decryptor); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index b2a8abb28b0c..c11a7468a712 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -33,6 +33,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -61,7 +62,7 @@ import java.util.Map; import java.util.Set; -import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; public class RepairDataFileScanUtil { private static final Logger logger = LoggerFactory.getLogger(RepairDataFileScanUtil.class); @@ -146,12 +147,13 @@ private void checkAlignedDeviceSeries( pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); } ByteBuffer pageData = chunkReader.readPageDataWithoutUncompressing(pageHeader); - + IDecryptor decryptor = IDecryptor.getDecryptor(timeChunk.getEncryptParam()); ByteBuffer uncompressedPageData = - uncompressPageData( + decryptAndUncompressPageData( pageHeader, IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()), - pageData); + pageData, + decryptor); Decoder decoder = Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); while (decoder.hasNext(uncompressedPageData)) { @@ -204,12 +206,13 @@ private void checkSingleNonAlignedSeries( pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); } ByteBuffer pageData = chunkReader.readPageDataWithoutUncompressing(pageHeader); - + IDecryptor decryptor = IDecryptor.getDecryptor(chunk.getEncryptParam()); ByteBuffer uncompressedPageData = - uncompressPageData( + decryptAndUncompressPageData( pageHeader, IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()), - pageData); + pageData, + decryptor); ByteBuffer timeBuffer = getTimeBufferFromNonAlignedPage(uncompressedPageData); Decoder timeDecoder = Decoder.getDecoderByType( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java index b231a4bc1185..2728b6a2823f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java @@ -21,6 +21,8 @@ import org.apache.iotdb.db.storageengine.dataregion.utils.SharedTimeDataBuffer; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; @@ -37,6 +39,8 @@ public class DiskAlignedChunkHandleImpl extends DiskChunkHandleImpl { private final SharedTimeDataBuffer sharedTimeDataBuffer; private int pageIndex = 0; + private EncryptParameter encryptParam; + public DiskAlignedChunkHandleImpl( IDeviceID deviceID, String measurement, @@ -53,13 +57,18 @@ public DiskAlignedChunkHandleImpl( protected void init(TsFileSequenceReader reader) throws IOException { sharedTimeDataBuffer.init(reader); super.init(reader); + this.encryptParam = reader.getEncryptParam(); } @Override public long[] getDataTime() throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer currentPageDataBuffer = ChunkReader.deserializePageData( - this.currentPageHeader, this.currentChunkDataBuffer, this.currentChunkHeader); + this.currentPageHeader, + this.currentChunkDataBuffer, + this.currentChunkHeader, + decryptor); int size = ReadWriteIOUtils.readInt(currentPageDataBuffer); byte[] bitmap = new byte[(size + 7) / 8]; currentPageDataBuffer.get(bitmap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java index 68ad659d90f5..4b147cad8ff8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java @@ -24,6 +24,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -46,6 +48,7 @@ public class DiskChunkHandleImpl implements IChunkHandle { private final IDeviceID deviceID; private final String measurement; private final String filePath; + private EncryptParameter encryptParam; protected ChunkHeader currentChunkHeader; protected PageHeader currentPageHeader; protected ByteBuffer currentChunkDataBuffer; @@ -81,6 +84,7 @@ protected void init(TsFileSequenceReader reader) throws IOException { Chunk chunk = reader.readMemChunk(offset); this.currentChunkDataBuffer = chunk.getData(); this.currentChunkHeader = chunk.getHeader(); + this.encryptParam = chunk.getEncryptParam(); } // Check if there is more pages to be scanned in Chunk. @@ -125,9 +129,10 @@ public long[] getPageStatisticsTime() { @Override public long[] getDataTime() throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer currentPageDataBuffer = ChunkReader.deserializePageData( - currentPageHeader, this.currentChunkDataBuffer, this.currentChunkHeader); + currentPageHeader, this.currentChunkDataBuffer, this.currentChunkHeader, decryptor); int timeBufferLength = ReadWriteForEncodingUtils.readUnsignedVarInt(currentPageDataBuffer); ByteBuffer timeBuffer = currentPageDataBuffer.slice(); timeBuffer.limit(timeBufferLength); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java index 8e01e85757eb..71420d44887b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java @@ -21,6 +21,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -45,6 +47,8 @@ public class SharedTimeDataBuffer { TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), TSDataType.INT64); + private EncryptParameter encryptParam; + public SharedTimeDataBuffer(IChunkMetadata timeChunkMetaData) { this.timeChunkMetaData = timeChunkMetaData; this.timeData = new ArrayList<>(); @@ -58,6 +62,7 @@ public void init(TsFileSequenceReader reader) throws IOException { Chunk timeChunk = reader.readMemChunk(timeChunkMetaData.getOffsetOfChunkHeader()); timeChunkHeader = timeChunk.getHeader(); timeBuffer = timeChunk.getData(); + encryptParam = timeChunk.getEncryptParam(); } public long[] getPageTime(int pageId) throws IOException { @@ -81,8 +86,9 @@ private void loadPageData() throws IOException { isSinglePageChunk() ? PageHeader.deserializeFrom(timeBuffer, timeChunkMetaData.getStatistics()) : PageHeader.deserializeFrom(timeBuffer, timeChunkHeader.getDataType()); + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer timePageData = - ChunkReader.deserializePageData(timePageHeader, timeBuffer, timeChunkHeader); + ChunkReader.deserializePageData(timePageHeader, timeBuffer, timeChunkHeader, decryptor); long[] pageData = new long[(int) timePageHeader.getNumOfValues()]; int index = 0; while (defaultTimeDecoder.hasNext(timePageData)) { diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 531891e456da..cdcd5a8571a4 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1536,6 +1536,21 @@ value_encoder=PLAIN # effectiveMode: hot_reload compressor=LZ4 +# Encryption configuration +# Data encryption function switch. +# effectiveMode: restart +encrypt_flag=false + +# Encryption configuration +# Data encryption method, supports org.apache.tsfile.encrypt.UNENCRYPTED, org.apache.tsfile.encrypt.AES128. +# effectiveMode: restart +encrypt_type=org.apache.tsfile.encrypt.UNENCRYPTED + +# Encryption configuration +# Data encryption key source. The key should be 16 byte String. +# effectiveMode: restart +encrypt_key_path= + #################### ### Authorization Configuration #################### diff --git a/pom.xml b/pom.xml index dfd1c7171ae7..ffd0ea756835 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ distribution example library-udf + integration-test @@ -166,7 +167,7 @@ 0.14.1 1.9 1.5.6-3 - 1.2.0-1c9924b4-SNAPSHOT + 1.2.0-241108-SNAPSHOT