From 70006df9bbc8044eb407b569a96d1d40bcb0359d Mon Sep 17 00:00:00 2001 From: Yang Yuming <50571168+YangYumings@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:56:11 +0800 Subject: [PATCH 1/6] When the thread is interrupted while retrying a connection, the loop will exit early. (#14062) --- .../java/org/apache/iotdb/session/SessionConnection.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index ef2b69baf6cb..e6f55f39286b 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -1109,7 +1109,13 @@ protected void withRetry(TFunction function) try { TimeUnit.MILLISECONDS.sleep(retryIntervalInMs); } catch (InterruptedException e) { - // just ignore + Thread.currentThread().interrupt(); + logger.warn( + "Thread {} was interrupted during retry {} with wait time {} ms. Exiting retry loop.", + Thread.currentThread().getName(), + i, + retryIntervalInMs); + break; } if (!reconnect()) { // reconnect failed, just continue to make another retry. From 57c668ca51608af92c0ccc1daafb14cc354cbc16 Mon Sep 17 00:00:00 2001 From: V_Galaxy Date: Wed, 13 Nov 2024 10:39:30 +0800 Subject: [PATCH 2/6] Pipe: atomically publish segment lock to avoid uninitialized volatile variable (#14064) --- .../resource/tsfile/PipeTsFileResourceSegmentLock.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceSegmentLock.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceSegmentLock.java index cc73ee0db38c..d1bf7fef2ad3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceSegmentLock.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceSegmentLock.java @@ -51,10 +51,13 @@ private void initIfNecessary() { lockSegmentSize = Math.min(SEGMENT_LOCK_MAX_SIZE, lockSegmentSize); lockSegmentSize = Math.max(SEGMENT_LOCK_MIN_SIZE, lockSegmentSize); - locks = new ReentrantLock[lockSegmentSize]; - for (int i = 0; i < locks.length; i++) { - locks[i] = new ReentrantLock(); + final ReentrantLock[] tmpLocks = new ReentrantLock[lockSegmentSize]; + for (int i = 0; i < tmpLocks.length; i++) { + tmpLocks[i] = new ReentrantLock(); } + + // publish this variable + locks = tmpLocks; } } } From e1cc2294e7d62dbac069aa44dcb8c5efc6fb52e2 Mon Sep 17 00:00:00 2001 From: jintao zhu <105690440+zhujt20@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:34:34 +0800 Subject: [PATCH 3/6] Add or modify encryption related codes (#13364) Co-authored-by: zhujt --- .../env/cluster/config/MppCommonConfig.java | 18 + .../cluster/config/MppSharedCommonConfig.java | 21 + .../env/remote/config/RemoteCommonConfig.java | 15 + .../apache/iotdb/itbase/env/CommonConfig.java | 6 + .../it/query/IoTDBEncryptionValueQueryIT.java | 676 ++++++++++++++++++ .../it/query/IoTDBLoadEncryptedTsFileIT.java | 204 ++++++ .../db/it/query/IoTDBLoadPlainTsFileIT.java | 145 ++++ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 18 + .../AlignedSinglePageWholeChunkReader.java | 11 +- .../scan/SinglePageWholeChunkReader.java | 50 +- .../load/LoadTsFileToTableModelAnalyzer.java | 10 + .../load/LoadTsFileToTreeModelAnalyzer.java | 10 + .../db/storageengine/buffer/ChunkCache.java | 12 +- .../reader/CompactionAlignedChunkReader.java | 18 +- .../fast/reader/CompactionChunkReader.java | 14 +- .../readchunk/loader/InstantChunkLoader.java | 3 +- .../readchunk/loader/InstantPageLoader.java | 17 +- .../repair/RepairDataFileScanUtil.java | 17 +- .../impl/DiskAlignedChunkHandleImpl.java | 11 +- .../filescan/impl/DiskChunkHandleImpl.java | 7 +- .../utils/SharedTimeDataBuffer.java | 8 +- .../conf/iotdb-system.properties.template | 15 + pom.xml | 1 + 23 files changed, 1273 insertions(+), 34 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 1760301ac737..9126e9cb1e8b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -97,6 +97,24 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptFlag(boolean encryptFlag) { + setProperty("encrypt_flag", String.valueOf(encryptFlag)); + return this; + } + + @Override + public CommonConfig setEncryptType(String encryptType) { + setProperty("encrypt_type", encryptType); + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + setProperty("encrypt_key_path", encryptKeyPath); + return this; + } + @Override public CommonConfig setUdfMemoryBudgetInMB(float udfCollectorMemoryBudgetInMB) { // udf_memory_budget_in_mb diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 969d4bb8d41c..b5598de6f693 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -75,6 +75,27 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptFlag(boolean encryptFlag) { + cnConfig.setProperty("encrypt_flag", String.valueOf(encryptFlag)); + dnConfig.setProperty("encrypt_flag", String.valueOf(encryptFlag)); + return this; + } + + @Override + public CommonConfig setEncryptType(String encryptType) { + cnConfig.setProperty("encrypt_type", encryptType); + dnConfig.setProperty("encrypt_type", encryptType); + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + cnConfig.setProperty("encrypt_key_path", encryptKeyPath); + dnConfig.setProperty("encrypt_key_path", encryptKeyPath); + return this; + } + @Override public CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs) { cnConfig.setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(maxMs); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index cee8aadf05f9..787b16139169 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -54,6 +54,21 @@ public CommonConfig setCompressor(String compressor) { return this; } + @Override + public CommonConfig setEncryptFlag(boolean encryptFlag) { + return this; + } + + @Override + public CommonConfig setEncryptType(String encryptType) { + return this; + } + + @Override + public CommonConfig setEncryptKeyPath(String encryptKeyPath) { + return this; + } + @Override public CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index d806be8db1dc..5dffdfe22e59 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -36,6 +36,12 @@ public interface CommonConfig { CommonConfig setCompressor(String compressor); + CommonConfig setEncryptFlag(boolean encryptFlag); + + CommonConfig setEncryptType(String encryptType); + + CommonConfig setEncryptKeyPath(String encryptKeyPath); + CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs); CommonConfig setUdfMemoryBudgetInMB(float udfCollectorMemoryBudgetInMB); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java new file mode 100644 index 000000000000..b63e409a3e31 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBEncryptionValueQueryIT.java @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBEncryptionValueQueryIT { + private static String[] sqls = + new String[] { + "CREATE DATABASE root.ln", + "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465600000,true)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465660000,true)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465720000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465780000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465840000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465900000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509465960000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466020000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466080000,false)", + "insert into root.ln.wf01.wt01(timestamp,status) values(1509466140000,false)", + "create timeseries root.ln.wf01.wt01.temperature with datatype=FLOAT,encoding=RLE", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600000,25.957603)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465660000,24.359503)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465720000,20.092794)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465780000,20.182663)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465840000,21.125198)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465900000,22.720892)", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465960000,20.71);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466020000,21.451046);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466080000,22.57987);", + "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509466140000,20.98177);", + "create timeseries root.ln.wf02.wt02.hardware with datatype=TEXT,encoding=PLAIN", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465600000,'v2')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465660000,'v2')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465720000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465780000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465840000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465900000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509465960000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466020000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466080000,'v1')", + "insert into root.ln.wf02.wt02(timestamp,hardware) values(1509466140000,'v1')", + "create timeseries root.ln.wf02.wt02.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465600000,true)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465660000,true)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465720000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465780000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465840000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465900000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509465960000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466020000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466080000,false)", + "insert into root.ln.wf02.wt02(timestamp,status) values(1509466140000,false)", + "flush", + "CREATE DATABASE root.sgcc", + "create timeseries root.sgcc.wf03.wt01.status with datatype=BOOLEAN,encoding=PLAIN", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465600000,true)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465660000,true)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465720000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465780000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465840000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465900000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509465960000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466020000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466080000,false)", + "insert into root.sgcc.wf03.wt01(timestamp,status) values(1509466140000,false)", + "create timeseries root.sgcc.wf03.wt01.temperature with datatype=FLOAT,encoding=RLE", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465600000,25.957603)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465660000,24.359503)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465720000,20.092794)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465780000,20.182663)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465840000,21.125198)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465900000,22.720892)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509465960000,20.71)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466020000,21.451046)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466080000,22.57987)", + "insert into root.sgcc.wf03.wt01(timestamp,temperature) values(1509466140000,20.98177)", + "flush", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEncryptFlag(true) + .setEncryptType("org.apache.tsfile.encrypt.AES128"); + EnvFactory.getEnv().initClusterEnvironment(); + importData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void importData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void selectTest() { + String[] retArray = + new String[] { + "1509465600000,true,25.96,v2,true,true,25.96,", + "1509465660000,true,24.36,v2,true,true,24.36,", + "1509465720000,false,20.09,v1,false,false,20.09,", + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + "1509466080000,false,22.58,v1,false,false,22.58,", + "1509466140000,false,20.98,v1,false,false,20.98,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = statement.executeQuery("select * from root.** where time>10"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void LimitTest() { + String[] retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // test 1: fetchSize < limitNumber + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + + ResultSet resultSet = + statement.executeQuery("select * from root.** where time>10 limit 5 offset 3"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + // test 1: fetchSize > limitNumber + statement.setFetchSize(10000); + Assert.assertEquals(10000, statement.getFetchSize()); + resultSet = statement.executeQuery("select * from root.** where time>10 limit 5 offset 3"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void InTest() { + String[] retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465840000,false,21.13,v1,false,false,21.13,", + "1509465900000,false,22.72,v1,false,false,22.72,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466020000,false,21.45,v1,false,false,21.45,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + // test 1: fetchSize < limitNumber + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + ResultSet resultSet = + statement.executeQuery( + "select * from root.** where time in (1509465780000, 1509465840000, 1509465900000, 1509465960000, 1509466020000)"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + retArray = + new String[] { + "1509465600000,true,25.96,v2,true,true,25.96,", + "1509465660000,true,24.36,v2,true,true,24.36,", + "1509465720000,false,20.09,v1,false,false,20.09,", + "1509466080000,false,22.58,v1,false,false,22.58,", + "1509466140000,false,20.98,v1,false,false,20.98,", + }; + resultSet = + statement.executeQuery( + "select * from root.** where time not in (1509465780000, 1509465840000, 1509465900000, 1509465960000, 1509466020000)"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(5, cnt); + + retArray = + new String[] { + "1509465780000,false,20.18,v1,false,false,20.18,", + "1509465960000,false,20.71,v1,false,false,20.71,", + "1509466080000,false,22.58,v1,false,false,22.58,", + }; + + resultSet = + statement.executeQuery( + "select * from root.** where root.ln.wf01.wt01.temperature in (20.18, 20.71, 22.58)"); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf01.wt01.status,root.ln.wf01.wt01.temperature," + + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,root.sgcc.wf03.wt01.status," + + "root.sgcc.wf03.wt01.temperature,", + new int[] { + Types.TIMESTAMP, + Types.BOOLEAN, + Types.FLOAT, + Types.VARCHAR, + Types.BOOLEAN, + Types.BOOLEAN, + Types.FLOAT, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } + + @Test + public void testRightTextQuery() { + // Text type uses the equal operator to query the correct result + String[] retArray = + new String[] { + "1509465600000,v2,", "1509465660000,v2,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + ResultSet resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.ln.wf02.wt02.hardware,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + resultSet = + statement.executeQuery("select hardware from root.ln.wf02.wt02 where hardware = 'v2'"); + resultSetMetaData = resultSet.getMetaData(); + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(2, cnt); + + } catch (Exception e) { + Assert.assertNull(e.getMessage()); + } + } + + @Test + public void RegexpTest() { + String[] retArray = + new String[] { + "1509465600000,v2,true,", "1509465660000,v2,true,", "1509465720000,v1,false,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.setFetchSize(4); + Assert.assertEquals(4, statement.getFetchSize()); + // Matches a string consisting of one lowercase letter and one digit. such as: "v1","v2" + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp '^[a-z][0-9]$' and time < 1509465780000"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + + retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp 'v*' "); + + resultSetMetaData = resultSet.getMetaData(); + actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(10, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void RegexpNonExistTest() { + + // Match nonexistent string.'s.' is indicates that the starting with s and the last is any + // single character + String[] retArray = + new String[] { + "1509465600000,v2,true,", + "1509465660000,v2,true,", + "1509465720000,v1,false,", + "1509465780000,v1,false,", + "1509465840000,v1,false,", + "1509465900000,v1,false,", + "1509465960000,v1,false,", + "1509466020000,v1,false,", + "1509466080000,v1,false,", + "1509466140000,v1,false,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + ResultSet resultSet = + statement.executeQuery( + "select hardware,status from root.ln.wf02.wt02 where hardware regexp 's.' "); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time," + "root.ln.wf02.wt02.hardware,root.ln.wf02.wt02.status,", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.BOOLEAN, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(0, cnt); + + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java new file mode 100644 index 000000000000..bbb2f173c36e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadEncryptedTsFileIT.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.it.env.EnvFactory; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IoTDBLoadEncryptedTsFileIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEncryptFlag(true) + .setEncryptType("org.apache.tsfile.encrypt.AES128"); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadSameWayEncryptedTsFileTest() { + String[] retArray = + new String[] { + "2,1,", "3,1,", "4,1,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg"); + statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateSameWayEncryptedFile(); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg.d1"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.testsg.d1.s1,", + new int[] { + Types.TIMESTAMP, Types.INTEGER, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void loadAnotherWayEncryptedTsFileTest() { + String unrecognizedType = "org.apache.tsfile.encrypt.SM4128"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg1"); + statement.execute("CREATE TIMESERIES root.testsg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateAnotherWayEncryptedFile(unrecognizedType); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg1.d1"); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("The encryption way of the TsFile is not supported.")); + } + } + + private File generateSameWayEncryptedFile() throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + config.setEncryptFlag("true"); + config.setEncryptType("org.apache.tsfile.encrypt.AES128"); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile, config)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema("s1", TSDataType.INT32), + EncryptUtils.getEncryptParameter(config)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + config.setEncryptFlag("false"); + config.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED"); + return tsfile; + } + + private File generateAnotherWayEncryptedFile(String unrecognizedType) throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + config.setEncryptFlag("true"); + config.setEncryptType("org.apache.tsfile.encrypt.AES128"); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile, config)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg1.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema("s1", TSDataType.INT32), + EncryptUtils.getEncryptParameter(config)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.setEncryptParam("2", unrecognizedType, EncryptUtils.getNormalKeyStr(config)); + writer.endFile(); + } + config.setEncryptFlag("false"); + config.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED"); + return tsfile; + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java new file mode 100644 index 000000000000..bdff3fb463c5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBLoadPlainTsFileIT.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.it.env.EnvFactory; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IoTDBLoadPlainTsFileIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEncryptFlag(true) + .setEncryptType("org.apache.tsfile.encrypt.AES128"); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadNormalTsFileTest() { + String[] retArray = + new String[] { + "2,1,", "3,1,", "4,1,", + }; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.tesgsg"); + statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN"); + File tsfile = generateNormalFile(); + statement.execute(String.format("load \"%s\"", tsfile.getParentFile().getAbsolutePath())); + ResultSet resultSet = statement.executeQuery("select s1 from root.testsg.d1"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,root.testsg.d1.s1,", + new int[] { + Types.TIMESTAMP, Types.INTEGER, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(3, cnt); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("The encryption way of the TsFile is not supported.")); + } + } + + private File generateNormalFile() throws IOException { + Path tempDir = Files.createTempDirectory(""); + tempDir.toFile().deleteOnExit(); + String tsfileName = + TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0); + File tsfile = new File(tempDir + File.separator + tsfileName); + Files.createFile(tsfile.toPath()); + + try (TsFileIOWriter writer = new TsFileIOWriter(tsfile)) { + writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT32)); + chunkWriter.write(2, 1); + chunkWriter.write(3, 1); + chunkWriter.write(4, 1); + chunkWriter.sealCurrentPage(); + + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + return tsfile; + } + + private List checkHeader( + ResultSetMetaData resultSetMetaData, String expectedHeaderStrings, int[] expectedTypes) + throws SQLException { + String[] expectedHeaders = expectedHeaderStrings.split(","); + Map expectedHeaderToTypeIndexMap = new HashMap<>(); + for (int i = 0; i < expectedHeaders.length; ++i) { + expectedHeaderToTypeIndexMap.put(expectedHeaders[i], i); + } + + List actualIndexToExpectedIndexList = new ArrayList<>(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + Integer typeIndex = expectedHeaderToTypeIndexMap.get(resultSetMetaData.getColumnName(i)); + Assert.assertNotNull(typeIndex); + Assert.assertEquals(expectedTypes[typeIndex], resultSetMetaData.getColumnType(i)); + actualIndexToExpectedIndexList.add(typeIndex); + } + return actualIndexToExpectedIndexList; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 9b8238c87e8d..d38688b0979d 100755 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -2464,6 +2464,24 @@ private void loadTsFileProps(Properties properties) throws IOException { ConfigurationFileUtils.getConfigurationDefaultValue("compressor"))) .map(String::trim) .orElse(ConfigurationFileUtils.getConfigurationDefaultValue("compressor"))); + TSFileDescriptor.getInstance() + .getConfig() + .setEncryptFlag( + properties.getProperty( + "encrypt_flag", + ConfigurationFileUtils.getConfigurationDefaultValue("encrypt_flag"))); + TSFileDescriptor.getInstance() + .getConfig() + .setEncryptType( + properties.getProperty( + "encrypt_type", + ConfigurationFileUtils.getConfigurationDefaultValue("encrypt_type"))); + TSFileDescriptor.getInstance() + .getConfig() + .setEncryptKeyFromPath( + properties.getProperty( + "encrypt_key_path", + ConfigurationFileUtils.getConfigurationDefaultValue("encrypt_key_path"))); TSFileDescriptor.getInstance() .getConfig() .setMaxTsBlockSizeInBytes( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java index 41bef7fe3b00..28e8a1deab4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/AlignedSinglePageWholeChunkReader.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -47,6 +49,8 @@ public class AlignedSinglePageWholeChunkReader extends AbstractChunkReader { // chunk data of the time column private final ByteBuffer timeChunkDataBuffer; + private final EncryptParameter encryptParam; + // chunk headers of all the sub sensors private final List valueChunkHeaderList = new ArrayList<>(); // chunk data of all the sub sensors @@ -59,6 +63,7 @@ public AlignedSinglePageWholeChunkReader(Chunk timeChunk, List valueChunk super(Long.MIN_VALUE, null); this.timeChunkHeader = timeChunk.getHeader(); this.timeChunkDataBuffer = timeChunk.getData(); + this.encryptParam = timeChunk.getEncryptParam(); valueChunkList.forEach( chunk -> { @@ -120,8 +125,10 @@ private void skipCurrentPage(PageHeader timePageHeader, List valuePa private AlignedPageReader constructAlignedPageReader( PageHeader timePageHeader, List rawValuePageHeaderList) throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer timePageData = - ChunkReader.deserializePageData(timePageHeader, timeChunkDataBuffer, timeChunkHeader); + ChunkReader.deserializePageData( + timePageHeader, timeChunkDataBuffer, timeChunkHeader, decryptor); List valuePageHeaderList = new ArrayList<>(); List valuePageDataList = new ArrayList<>(); @@ -143,7 +150,7 @@ private AlignedPageReader constructAlignedPageReader( valuePageHeaderList.add(valuePageHeader); valuePageDataList.add( ChunkReader.deserializePageData( - valuePageHeader, valueChunkDataBufferList.get(i), valueChunkHeader)); + valuePageHeader, valueChunkDataBufferList.get(i), valueChunkHeader, decryptor)); valueDataTypeList.add(valueChunkHeader.getDataType()); valueDecoderList.add( Decoder.getDecoderByType( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java index cd85ae7866d5..ed831644988f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java @@ -21,8 +21,11 @@ import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.reader.chunk.AbstractChunkReader; @@ -32,16 +35,19 @@ import java.io.Serializable; import java.nio.ByteBuffer; +import static org.apache.tsfile.file.metadata.enums.CompressionType.UNCOMPRESSED; + public class SinglePageWholeChunkReader extends AbstractChunkReader { private final ChunkHeader chunkHeader; private final ByteBuffer chunkDataBuffer; + private final EncryptParameter encryptParam; public SinglePageWholeChunkReader(Chunk chunk) throws IOException { super(Long.MIN_VALUE, null); this.chunkHeader = chunk.getHeader(); this.chunkDataBuffer = chunk.getData(); - + this.encryptParam = chunk.getEncryptParam(); initAllPageReaders(); } @@ -56,9 +62,10 @@ private void initAllPageReaders() throws IOException { } private PageReader constructPageReader(PageHeader pageHeader) throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); return new PageReader( pageHeader, - deserializePageData(pageHeader, chunkDataBuffer, chunkHeader), + deserializePageData(pageHeader, chunkDataBuffer, chunkHeader, decryptor), chunkHeader.getDataType(), Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), defaultTimeDecoder, @@ -88,11 +95,14 @@ public static ByteBuffer readCompressedPageData(PageHeader pageHeader, ByteBuffe public static ByteBuffer uncompressPageData( PageHeader pageHeader, IUnCompressor unCompressor, ByteBuffer compressedPageData) throws IOException { + if (unCompressor.getCodecName() == UNCOMPRESSED) { + return compressedPageData; + } int compressedPageBodyLength = pageHeader.getCompressedSize(); - byte[] uncompressedPageData = new byte[pageHeader.getUncompressedSize()]; + ByteBuffer uncompressedPageData = ByteBuffer.allocate(pageHeader.getUncompressedSize()); try { unCompressor.uncompress( - compressedPageData.array(), 0, compressedPageBodyLength, uncompressedPageData, 0); + compressedPageData.array(), 0, compressedPageBodyLength, uncompressedPageData.array(), 0); } catch (Exception e) { throw new IOException( "Uncompress error! uncompress size: " @@ -101,16 +111,40 @@ public static ByteBuffer uncompressPageData( + pageHeader.getCompressedSize() + "page header: " + pageHeader - + e.getMessage()); + + e.getMessage(), + e); } - return ByteBuffer.wrap(uncompressedPageData); + return uncompressedPageData; + } + + public static ByteBuffer decrypt(IDecryptor decryptor, ByteBuffer buffer) { + if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return buffer; + } + return ByteBuffer.wrap( + decryptor.decrypt( + buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining())); + } + + public static ByteBuffer decryptAndUncompressPageData( + PageHeader pageHeader, + IUnCompressor unCompressor, + ByteBuffer compressedPageData, + IDecryptor decryptor) + throws IOException { + return uncompressPageData(pageHeader, unCompressor, decrypt(decryptor, compressedPageData)); } public static ByteBuffer deserializePageData( - PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader) throws IOException { + PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader, IDecryptor decryptor) + throws IOException { IUnCompressor unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()); ByteBuffer compressedPageBody = readCompressedPageData(pageHeader, chunkBuffer); - return uncompressPageData(pageHeader, unCompressor, compressedPageBody); + if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return uncompressPageData(pageHeader, unCompressor, compressedPageBody); + } else { + return decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageBody, decryptor); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java index 8138b9394271..24efd6e567ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java @@ -44,6 +44,8 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.commons.io.FileUtils; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -53,6 +55,7 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -134,6 +137,13 @@ protected void analyzeSingleTsFile(final File tsFile) throw new SemanticException("Attempted to load a tree-model TsFile into table-model."); } + // check whether the encrypt type of the tsfile is supported + EncryptParameter param = reader.getEncryptParam(); + if (!Objects.equals(param.getType(), EncryptUtils.encryptParam.getType()) + || !Arrays.equals(param.getKey(), EncryptUtils.encryptParam.getKey())) { + throw new SemanticException("The encryption way of the TsFile is not supported."); + } + // construct tsfile resource final TsFileResource tsFileResource = constructTsFileResource(reader, tsFile); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java index f07cee07f63c..a75c9a87bbe5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java @@ -37,6 +37,8 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.commons.io.FileUtils; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -46,6 +48,7 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -118,6 +121,13 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx throw new LoadEmptyFileException(tsFile.getAbsolutePath()); } + // check whether the encrypt type of the tsfile is supported + EncryptParameter param = reader.getEncryptParam(); + if (!Objects.equals(param.getType(), EncryptUtils.encryptParam.getType()) + || !Arrays.equals(param.getKey(), EncryptUtils.encryptParam.getKey())) { + throw new SemanticException("The encryption way of the TsFile is not supported."); + } + // check whether the tsfile is tree-model or not // TODO: currently, loading a file with both tree-model and table-model data is not supported. // May need to support this and remove this check in the future. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java index 3ad81d399309..0a12a529abf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java @@ -119,7 +119,11 @@ public Chunk get( FileReaderManager.getInstance().get(chunkCacheKey.getFilePath(), true); Chunk chunk = reader.readMemChunk(chunkCacheKey.offsetOfChunkHeader); return new Chunk( - chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic); + chunk.getHeader(), + chunk.getData().duplicate(), + timeRangeList, + chunkStatistic, + chunk.getEncryptParam()); } Chunk chunk = lruCache.get(chunkCacheKey); @@ -129,7 +133,11 @@ public Chunk get( } return new Chunk( - chunk.getHeader(), chunk.getData().duplicate(), timeRangeList, chunkStatistic); + chunk.getHeader(), + chunk.getData().duplicate(), + timeRangeList, + chunkStatistic, + chunk.getEncryptParam()); } finally { SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( READ_CHUNK_ALL, System.nanoTime() - startTime); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java index ff41e5e840e7..a94150deca19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java @@ -24,6 +24,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -39,7 +41,7 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; public class CompactionAlignedChunkReader { @@ -47,6 +49,8 @@ public class CompactionAlignedChunkReader { private final List valueChunkHeaderList = new ArrayList<>(); private final IUnCompressor timeUnCompressor; + + private final EncryptParameter encryptParam; private final boolean ignoreAllNullRows; private final Decoder timeDecoder = Decoder.getDecoderByType( @@ -65,6 +69,7 @@ public CompactionAlignedChunkReader( Chunk timeChunk, List valueChunkList, boolean ignoreAllNullRows) { ChunkHeader timeChunkHeader = timeChunk.getHeader(); this.timeUnCompressor = IUnCompressor.getUnCompressor(timeChunkHeader.getCompressionType()); + this.encryptParam = timeChunk.getEncryptParam(); this.timeDeleteIntervalList = timeChunk.getDeleteIntervalList(); this.valueDeleteIntervalList = new ArrayList<>(valueChunkList.size()); @@ -113,9 +118,11 @@ private IPointReader getPontReader( boolean ignoreAllNullRows) throws IOException { - // uncompress time page data + // decrypt and uncompress time page data + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer uncompressedTimePageData = - uncompressPageData(timePageHeader, timeUnCompressor, compressedTimePageData); + decryptAndUncompressPageData( + timePageHeader, timeUnCompressor, compressedTimePageData, decryptor); TimePageReader timePageReader = new TimePageReader(timePageHeader, uncompressedTimePageData, timeDecoder); timePageReader.setDeleteIntervalList(timeDeleteIntervalList); @@ -128,10 +135,11 @@ private IPointReader getPontReader( } else { ChunkHeader valueChunkHeader = valueChunkHeaderList.get(i); ByteBuffer uncompressedPageData = - uncompressPageData( + decryptAndUncompressPageData( valuePageHeaders.get(i), IUnCompressor.getUnCompressor(valueChunkHeader.getCompressionType()), - compressedValuePageDatas.get(i)); + compressedValuePageDatas.get(i), + decryptor); TSDataType valueType = valueChunkHeader.getDataType(); ValuePageReader valuePageReader = new ValuePageReader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java index 81836b43ea49..2b39064d36a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java @@ -22,6 +22,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -39,13 +41,15 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; public class CompactionChunkReader { private final ChunkHeader chunkHeader; private ByteBuffer chunkDataBuffer; private final IUnCompressor unCompressor; + + private final EncryptParameter encryptParam; private final Decoder timeDecoder = Decoder.getDecoderByType( TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), @@ -66,6 +70,7 @@ public CompactionChunkReader(Chunk chunk) { this.unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()); this.deleteIntervalList = chunk.getDeleteIntervalList(); this.chunkStatistic = chunk.getChunkStatistic(); + this.encryptParam = chunk.getEncryptParam(); } /** @@ -126,9 +131,10 @@ public static ByteBuffer readCompressedPageData(PageHeader pageHeader, ByteBuffe */ public TsBlock readPageData(PageHeader pageHeader, ByteBuffer compressedPageData) throws IOException { - // uncompress page data - ByteBuffer pageData = uncompressPageData(pageHeader, unCompressor, compressedPageData); - + // decrypt and uncompress page data + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); + ByteBuffer pageData = + decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageData, decryptor); // decode page data TSDataType dataType = chunkHeader.getDataType(); Decoder valueDecoder = Decoder.getDecoderByType(chunkHeader.getEncodingType(), dataType); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java index 41d88fc3d79b..5256204b5e4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantChunkLoader.java @@ -93,7 +93,8 @@ public List getPages() { chunk.getHeader().getDataType(), chunk.getHeader().getEncodingType(), chunkMetadata, - pageModifiedStatus)); + pageModifiedStatus, + chunk.getEncryptParam())); } return pageList; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java index 8da39abda78c..c01ed64e8b68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/loader/InstantPageLoader.java @@ -22,23 +22,29 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; public class InstantPageLoader extends PageLoader { private ByteBuffer pageData; + private EncryptParameter encryptParam; + public InstantPageLoader() {} public InstantPageLoader( @@ -49,9 +55,11 @@ public InstantPageLoader( TSDataType dataType, TSEncoding encoding, ChunkMetadata chunkMetadata, - ModifiedStatus modifiedStatus) { + ModifiedStatus modifiedStatus, + EncryptParameter encryptParam) { super(file, pageHeader, compressionType, dataType, encoding, chunkMetadata, modifiedStatus); this.pageData = pageData; + this.encryptParam = encryptParam; } @Override @@ -62,7 +70,12 @@ public ByteBuffer getCompressedData() { @Override public ByteBuffer getUnCompressedData() throws IOException { IUnCompressor unCompressor = IUnCompressor.getUnCompressor(compressionType); - return uncompressPageData(pageHeader, unCompressor, pageData); + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); + if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return uncompressPageData(pageHeader, unCompressor, pageData); + } else { + return decryptAndUncompressPageData(pageHeader, unCompressor, pageData, decryptor); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index b2a8abb28b0c..c11a7468a712 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -33,6 +33,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -61,7 +62,7 @@ import java.util.Map; import java.util.Set; -import static org.apache.tsfile.read.reader.chunk.ChunkReader.uncompressPageData; +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; public class RepairDataFileScanUtil { private static final Logger logger = LoggerFactory.getLogger(RepairDataFileScanUtil.class); @@ -146,12 +147,13 @@ private void checkAlignedDeviceSeries( pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); } ByteBuffer pageData = chunkReader.readPageDataWithoutUncompressing(pageHeader); - + IDecryptor decryptor = IDecryptor.getDecryptor(timeChunk.getEncryptParam()); ByteBuffer uncompressedPageData = - uncompressPageData( + decryptAndUncompressPageData( pageHeader, IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()), - pageData); + pageData, + decryptor); Decoder decoder = Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); while (decoder.hasNext(uncompressedPageData)) { @@ -204,12 +206,13 @@ private void checkSingleNonAlignedSeries( pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); } ByteBuffer pageData = chunkReader.readPageDataWithoutUncompressing(pageHeader); - + IDecryptor decryptor = IDecryptor.getDecryptor(chunk.getEncryptParam()); ByteBuffer uncompressedPageData = - uncompressPageData( + decryptAndUncompressPageData( pageHeader, IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()), - pageData); + pageData, + decryptor); ByteBuffer timeBuffer = getTimeBufferFromNonAlignedPage(uncompressedPageData); Decoder timeDecoder = Decoder.getDecoderByType( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java index b231a4bc1185..2728b6a2823f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java @@ -21,6 +21,8 @@ import org.apache.iotdb.db.storageengine.dataregion.utils.SharedTimeDataBuffer; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; @@ -37,6 +39,8 @@ public class DiskAlignedChunkHandleImpl extends DiskChunkHandleImpl { private final SharedTimeDataBuffer sharedTimeDataBuffer; private int pageIndex = 0; + private EncryptParameter encryptParam; + public DiskAlignedChunkHandleImpl( IDeviceID deviceID, String measurement, @@ -53,13 +57,18 @@ public DiskAlignedChunkHandleImpl( protected void init(TsFileSequenceReader reader) throws IOException { sharedTimeDataBuffer.init(reader); super.init(reader); + this.encryptParam = reader.getEncryptParam(); } @Override public long[] getDataTime() throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer currentPageDataBuffer = ChunkReader.deserializePageData( - this.currentPageHeader, this.currentChunkDataBuffer, this.currentChunkHeader); + this.currentPageHeader, + this.currentChunkDataBuffer, + this.currentChunkHeader, + decryptor); int size = ReadWriteIOUtils.readInt(currentPageDataBuffer); byte[] bitmap = new byte[(size + 7) / 8]; currentPageDataBuffer.get(bitmap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java index 68ad659d90f5..4b147cad8ff8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java @@ -24,6 +24,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -46,6 +48,7 @@ public class DiskChunkHandleImpl implements IChunkHandle { private final IDeviceID deviceID; private final String measurement; private final String filePath; + private EncryptParameter encryptParam; protected ChunkHeader currentChunkHeader; protected PageHeader currentPageHeader; protected ByteBuffer currentChunkDataBuffer; @@ -81,6 +84,7 @@ protected void init(TsFileSequenceReader reader) throws IOException { Chunk chunk = reader.readMemChunk(offset); this.currentChunkDataBuffer = chunk.getData(); this.currentChunkHeader = chunk.getHeader(); + this.encryptParam = chunk.getEncryptParam(); } // Check if there is more pages to be scanned in Chunk. @@ -125,9 +129,10 @@ public long[] getPageStatisticsTime() { @Override public long[] getDataTime() throws IOException { + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer currentPageDataBuffer = ChunkReader.deserializePageData( - currentPageHeader, this.currentChunkDataBuffer, this.currentChunkHeader); + currentPageHeader, this.currentChunkDataBuffer, this.currentChunkHeader, decryptor); int timeBufferLength = ReadWriteForEncodingUtils.readUnsignedVarInt(currentPageDataBuffer); ByteBuffer timeBuffer = currentPageDataBuffer.slice(); timeBuffer.limit(timeBufferLength); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java index 8e01e85757eb..71420d44887b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java @@ -21,6 +21,8 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -45,6 +47,8 @@ public class SharedTimeDataBuffer { TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), TSDataType.INT64); + private EncryptParameter encryptParam; + public SharedTimeDataBuffer(IChunkMetadata timeChunkMetaData) { this.timeChunkMetaData = timeChunkMetaData; this.timeData = new ArrayList<>(); @@ -58,6 +62,7 @@ public void init(TsFileSequenceReader reader) throws IOException { Chunk timeChunk = reader.readMemChunk(timeChunkMetaData.getOffsetOfChunkHeader()); timeChunkHeader = timeChunk.getHeader(); timeBuffer = timeChunk.getData(); + encryptParam = timeChunk.getEncryptParam(); } public long[] getPageTime(int pageId) throws IOException { @@ -81,8 +86,9 @@ private void loadPageData() throws IOException { isSinglePageChunk() ? PageHeader.deserializeFrom(timeBuffer, timeChunkMetaData.getStatistics()) : PageHeader.deserializeFrom(timeBuffer, timeChunkHeader.getDataType()); + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); ByteBuffer timePageData = - ChunkReader.deserializePageData(timePageHeader, timeBuffer, timeChunkHeader); + ChunkReader.deserializePageData(timePageHeader, timeBuffer, timeChunkHeader, decryptor); long[] pageData = new long[(int) timePageHeader.getNumOfValues()]; int index = 0; while (defaultTimeDecoder.hasNext(timePageData)) { diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 531891e456da..cdcd5a8571a4 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1536,6 +1536,21 @@ value_encoder=PLAIN # effectiveMode: hot_reload compressor=LZ4 +# Encryption configuration +# Data encryption function switch. +# effectiveMode: restart +encrypt_flag=false + +# Encryption configuration +# Data encryption method, supports org.apache.tsfile.encrypt.UNENCRYPTED, org.apache.tsfile.encrypt.AES128. +# effectiveMode: restart +encrypt_type=org.apache.tsfile.encrypt.UNENCRYPTED + +# Encryption configuration +# Data encryption key source. The key should be 16 byte String. +# effectiveMode: restart +encrypt_key_path= + #################### ### Authorization Configuration #################### diff --git a/pom.xml b/pom.xml index 8738d197a28b..b2f76c721476 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ distribution example library-udf + integration-test From 34221fb035fa5b2df4a2ecbcaafd64d292927a49 Mon Sep 17 00:00:00 2001 From: V_Galaxy Date: Wed, 13 Nov 2024 12:35:08 +0800 Subject: [PATCH 4/6] Subscription: avoid the file reading when the memory is not enough for tsfile slicing & implement all-out effort global timeout control & support client resume from breakpoint for tsfile consumption (#13992) --- .../SubscriptionPipeTimeoutException.java | 2 +- .../SubscriptionPollTimeoutException.java | 45 +++ .../SubscriptionTimeoutException.java | 47 +++ .../payload/poll/SubscriptionPollRequest.java | 2 +- .../consumer/SubscriptionConsumer.java | 173 ++++++----- .../SubscriptionExecutorServiceManager.java | 11 +- .../consumer/SubscriptionProvider.java | 13 +- .../processor/PipeProcessorSubtask.java | 4 +- .../evolvable/batch/PipeTabletEventBatch.java | 5 +- .../tsfile/PipeTsFileInsertionEvent.java | 55 ++-- .../resource/memory/PipeMemoryManager.java | 6 + .../agent/SubscriptionReceiverAgent.java | 13 + .../broker/SubscriptionPrefetchingQueue.java | 22 +- .../SubscriptionPrefetchingTabletQueue.java | 22 +- .../SubscriptionPrefetchingTsFileQueue.java | 35 +-- .../subscription/event/SubscriptionEvent.java | 6 +- .../batch/SubscriptionPipeEventBatch.java | 6 +- .../SubscriptionPipeTabletEventBatch.java | 6 +- .../SubscriptionPipeTsFileEventBatch.java | 8 +- .../SubscriptionEventExtendableResponse.java | 10 - .../response/SubscriptionEventResponse.java | 4 +- .../SubscriptionEventSingleResponse.java | 2 +- .../SubscriptionEventTabletResponse.java | 10 + .../SubscriptionEventTsFileResponse.java | 73 ++++- .../receiver/SubscriptionReceiver.java | 2 + .../receiver/SubscriptionReceiverV1.java | 271 ++++++++++-------- .../SubscriptionSubtaskExecutor.java | 50 ++++ .../SubscriptionSubtaskScheduler.java | 39 +++ .../subtask/SubscriptionConnectorSubtask.java | 44 ++- .../subtask/SubscriptionReceiverSubtask.java | 24 ++ .../iotdb/commons/conf/CommonConfig.java | 22 +- .../iotdb/commons/conf/CommonDescriptor.java | 11 +- .../task/execution/PipeSubtaskExecutor.java | 32 ++- .../subtask/PipeAbstractConnectorSubtask.java | 12 +- .../pipe/agent/task/subtask/PipeSubtask.java | 4 +- .../ref/PipePhantomReferenceManager.java | 4 +- .../config/SubscriptionConfig.java | 14 +- 37 files changed, 788 insertions(+), 321 deletions(-) create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskScheduler.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionReceiverSubtask.java diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java index f9ecc0f1b8b8..26231e483969 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPipeTimeoutException.java @@ -21,7 +21,7 @@ import java.util.Objects; -public class SubscriptionPipeTimeoutException extends SubscriptionRuntimeNonCriticalException { +public class SubscriptionPipeTimeoutException extends SubscriptionTimeoutException { public SubscriptionPipeTimeoutException(final String message) { super(message); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java new file mode 100644 index 000000000000..67c460ecfb30 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionPollTimeoutException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public class SubscriptionPollTimeoutException extends SubscriptionTimeoutException { + + public SubscriptionPollTimeoutException(final String message) { + super(message); + } + + public SubscriptionPollTimeoutException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionPollTimeoutException + && Objects.equals(getMessage(), ((SubscriptionPollTimeoutException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((SubscriptionPollTimeoutException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java new file mode 100644 index 000000000000..f6471a74abaf --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/SubscriptionTimeoutException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.exception; + +import java.util.Objects; + +public abstract class SubscriptionTimeoutException extends SubscriptionRuntimeNonCriticalException { + + public static String KEYWORD = "TimeoutException"; + + public SubscriptionTimeoutException(final String message) { + super(message); + } + + public SubscriptionTimeoutException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public boolean equals(final Object obj) { + return obj instanceof SubscriptionTimeoutException + && Objects.equals(getMessage(), ((SubscriptionTimeoutException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((SubscriptionTimeoutException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java index b9337e5779fe..3337887b185f 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java @@ -36,7 +36,7 @@ public class SubscriptionPollRequest { private final transient SubscriptionPollPayload payload; - private final transient long timeoutMs; // unused now + private final transient long timeoutMs; /** The maximum size, in bytes, for the response payload. */ private final transient long maxBytes; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 1bf0aea9c1ed..70738a5aece5 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -26,8 +26,10 @@ import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionPipeTimeoutException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionPollTimeoutException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionTimeoutException; import org.apache.iotdb.rpc.subscription.payload.poll.ErrorPayload; import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload; import org.apache.iotdb.rpc.subscription.payload.poll.FilePiecePayload; @@ -111,9 +113,9 @@ abstract class SubscriptionConsumer implements AutoCloseable { private final String fileSaveDir; private final boolean fileSaveFsync; + private final Set inFlightFilesCommitContextSet = new HashSet<>(); private final int thriftMaxFrameSize; - private final int maxPollParallelism; @SuppressWarnings("java:S3077") @@ -171,7 +173,6 @@ protected SubscriptionConsumer(final Builder builder) { this.fileSaveFsync = builder.fileSaveFsync; this.thriftMaxFrameSize = builder.thriftMaxFrameSize; - this.maxPollParallelism = builder.maxPollParallelism; } @@ -399,33 +400,47 @@ private Path getFileDir(final String topicName) throws IOException { } private Path getFilePath( + final SubscriptionCommitContext commitContext, final String topicName, final String fileName, final boolean allowFileAlreadyExistsException, final boolean allowInvalidPathException) throws SubscriptionException { try { - final Path filePath = getFileDir(topicName).resolve(fileName); - Files.createFile(filePath); - return filePath; - } catch (final FileAlreadyExistsException fileAlreadyExistsException) { - if (allowFileAlreadyExistsException) { - final String suffix = RandomStringGenerator.generate(16); - LOGGER.warn( - "Detect already existed file {} when polling topic {}, add random suffix {} to filename", - fileName, - topicName, - suffix); - return getFilePath(topicName, fileName + "." + suffix, false, true); + final Path filePath; + try { + filePath = getFileDir(topicName).resolve(fileName); + } catch (final InvalidPathException invalidPathException) { + if (allowInvalidPathException) { + return getFilePath(commitContext, URLEncoder.encode(topicName), fileName, true, false); + } + throw new SubscriptionRuntimeNonCriticalException( + invalidPathException.getMessage(), invalidPathException); } - throw new SubscriptionRuntimeNonCriticalException( - fileAlreadyExistsException.getMessage(), fileAlreadyExistsException); - } catch (final InvalidPathException invalidPathException) { - if (allowInvalidPathException) { - return getFilePath(URLEncoder.encode(topicName), fileName, true, false); + + try { + Files.createFile(filePath); + return filePath; + } catch (final FileAlreadyExistsException fileAlreadyExistsException) { + if (allowFileAlreadyExistsException) { + if (inFlightFilesCommitContextSet.contains(commitContext)) { + LOGGER.info( + "Detect already existed file {} when polling topic {}, resume consumption", + fileName, + topicName); + return filePath; + } + final String suffix = RandomStringGenerator.generate(16); + LOGGER.warn( + "Detect already existed file {} when polling topic {}, add random suffix {} to filename", + fileName, + topicName, + suffix); + return getFilePath(commitContext, topicName, fileName + "." + suffix, false, true); + } + throw new SubscriptionRuntimeNonCriticalException( + fileAlreadyExistsException.getMessage(), fileAlreadyExistsException); } - throw new SubscriptionRuntimeNonCriticalException( - invalidPathException.getMessage(), invalidPathException); } catch (final IOException e) { throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } @@ -579,7 +594,7 @@ private List singlePoll( final List currentMessages = new ArrayList<>(); try { currentResponses.clear(); - currentResponses = pollInternal(topicNames); + currentResponses = pollInternal(topicNames, timer.remainingMs()); for (final SubscriptionPollResponse response : currentResponses) { final short responseType = response.getResponseType(); if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) { @@ -676,11 +691,14 @@ private Optional pollFile( final SubscriptionCommitContext commitContext = response.getCommitContext(); final String fileName = ((FileInitPayload) response.getPayload()).getFileName(); final String topicName = commitContext.getTopicName(); - final Path filePath = getFilePath(topicName, fileName, true, true); + final Path filePath = getFilePath(commitContext, topicName, fileName, true, true); final File file = filePath.toFile(); try (final RandomAccessFile fileWriter = new RandomAccessFile(file, "rw")) { return Optional.of(pollFileInternal(commitContext, fileName, file, fileWriter, timer)); } catch (final Exception e) { + if (!(e instanceof SubscriptionPollTimeoutException)) { + inFlightFilesCommitContextSet.remove(commitContext); + } // construct temporary message to nack nack( Collections.singletonList( @@ -696,26 +714,31 @@ private SubscriptionMessage pollFileInternal( final RandomAccessFile fileWriter, final PollTimer timer) throws IOException, SubscriptionException { + long writingOffset = fileWriter.length(); + LOGGER.info( - "{} start to poll file {} with commit context {}", + "{} start to poll file {} with commit context {} at offset {}", this, file.getAbsolutePath(), - commitContext); + commitContext, + writingOffset); - long writingOffset = fileWriter.length(); + fileWriter.seek(writingOffset); while (true) { timer.update(); - if (timer.isExpired()) { - final String errorMessage = + if (timer.isExpired(TIMER_DELTA_MS)) { + // resume from breakpoint if timeout happened when polling files + inFlightFilesCommitContextSet.add(commitContext); + final String message = String.format( - "timeout while poll file %s with commit context: %s, consumer: %s", - file.getAbsolutePath(), commitContext, this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); + "Timeout occurred when SubscriptionConsumer %s polling file %s with commit context %s, record writing offset %s for subsequent poll", + this, file.getAbsolutePath(), commitContext, writingOffset); + LOGGER.info(message); + throw new SubscriptionRuntimeNonCriticalException(message); } final List responses = - pollFileInternal(commitContext, writingOffset); + pollFileInternal(commitContext, writingOffset, timer.remainingMs()); // It's agreed that the server will always return at least one response, even in case of // failure. @@ -831,6 +854,7 @@ private SubscriptionMessage pollFileInternal( commitContext); // generate subscription message + inFlightFilesCommitContextSet.remove(commitContext); return new SubscriptionMessage(commitContext, file.getAbsolutePath()); } case ERROR: @@ -839,17 +863,30 @@ private SubscriptionMessage pollFileInternal( final String errorMessage = ((ErrorPayload) payload).getErrorMessage(); final boolean critical = ((ErrorPayload) payload).isCritical(); - LOGGER.warn( - "Error occurred when SubscriptionConsumer {} polling file {} with commit context {}: {}, critical: {}", - this, - file.getAbsolutePath(), - commitContext, - errorMessage, - critical); - if (critical) { - throw new SubscriptionRuntimeCriticalException(errorMessage); + if (!critical + && Objects.nonNull(errorMessage) + && errorMessage.contains(SubscriptionTimeoutException.KEYWORD)) { + // resume from breakpoint if timeout happened when polling files + inFlightFilesCommitContextSet.add(commitContext); + final String message = + String.format( + "Timeout occurred when SubscriptionConsumer %s polling file %s with commit context %s, record writing offset %s for subsequent poll", + this, file.getAbsolutePath(), commitContext, writingOffset); + LOGGER.info(message); + throw new SubscriptionPollTimeoutException(message); } else { - throw new SubscriptionRuntimeNonCriticalException(errorMessage); + LOGGER.warn( + "Error occurred when SubscriptionConsumer {} polling file {} with commit context {}: {}, critical: {}", + this, + file.getAbsolutePath(), + commitContext, + errorMessage, + critical); + if (critical) { + throw new SubscriptionRuntimeCriticalException(errorMessage); + } else { + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } } } default: @@ -880,16 +917,6 @@ private Optional pollTabletsInternal( int nextOffset = ((TabletsPayload) initialResponse.getPayload()).getNextOffset(); while (true) { - timer.update(); - if (timer.isExpired()) { - final String errorMessage = - String.format( - "timeout while poll tablets with commit context: %s, consumer: %s", - commitContext, this); - LOGGER.warn(errorMessage); - throw new SubscriptionRuntimeNonCriticalException(errorMessage); - } - if (nextOffset < 0) { if (!Objects.equals(tablets.size(), -nextOffset)) { final String errorMessage = @@ -902,8 +929,18 @@ private Optional pollTabletsInternal( return Optional.of(new SubscriptionMessage(commitContext, tablets)); } + timer.update(); + if (timer.isExpired(TIMER_DELTA_MS)) { + final String errorMessage = + String.format( + "timeout while poll tablets with commit context: %s, consumer: %s", + commitContext, this); + LOGGER.warn(errorMessage); + throw new SubscriptionRuntimeNonCriticalException(errorMessage); + } + final List responses = - pollTabletsInternal(commitContext, nextOffset); + pollTabletsInternal(commitContext, nextOffset, timer.remainingMs()); // It's agreed that the server will always return at least one response, even in case of // failure. @@ -972,8 +1009,8 @@ private Optional pollTabletsInternal( } } - private List pollInternal(final Set topicNames) - throws SubscriptionException { + private List pollInternal( + final Set topicNames, final long timeoutMs) throws SubscriptionException { providers.acquireReadLock(); try { final SubscriptionProvider provider = providers.getNextAvailableProvider(); @@ -988,7 +1025,7 @@ private List pollInternal(final Set topicNames } // ignore SubscriptionConnectionException to improve poll auto retry try { - return provider.poll(topicNames); + return provider.poll(topicNames, timeoutMs); } catch (final SubscriptionConnectionException ignored) { return Collections.emptyList(); } @@ -998,7 +1035,7 @@ private List pollInternal(final Set topicNames } private List pollFileInternal( - final SubscriptionCommitContext commitContext, final long writingOffset) + final SubscriptionCommitContext commitContext, final long writingOffset, final long timeoutMs) throws SubscriptionException { final int dataNodeId = commitContext.getDataNodeId(); providers.acquireReadLock(); @@ -1015,7 +1052,7 @@ private List pollFileInternal( } // ignore SubscriptionConnectionException to improve poll auto retry try { - return provider.pollFile(commitContext, writingOffset); + return provider.pollFile(commitContext, writingOffset, timeoutMs); } catch (final SubscriptionConnectionException ignored) { return Collections.emptyList(); } @@ -1025,7 +1062,7 @@ private List pollFileInternal( } private List pollTabletsInternal( - final SubscriptionCommitContext commitContext, final int offset) + final SubscriptionCommitContext commitContext, final int offset, final long timeoutMs) throws SubscriptionException { final int dataNodeId = commitContext.getDataNodeId(); providers.acquireReadLock(); @@ -1042,7 +1079,7 @@ private List pollTabletsInternal( } // ignore SubscriptionConnectionException to improve poll auto retry try { - return provider.pollTablets(commitContext, offset); + return provider.pollTablets(commitContext, offset, timeoutMs); } catch (final SubscriptionConnectionException ignored) { return Collections.emptyList(); } @@ -1061,7 +1098,7 @@ protected void ack(final Iterable messages) throws Subscrip .computeIfAbsent(message.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) .add(message.getCommitContext()); } - for (final Map.Entry> entry : + for (final Entry> entry : dataNodeIdToSubscriptionCommitContexts.entrySet()) { commitInternal(entry.getKey(), entry.getValue(), false); } @@ -1073,7 +1110,10 @@ protected void nack(final Iterable messages) throws Subscri for (final SubscriptionMessage message : messages) { // make every effort to delete stale intermediate file if (Objects.equals( - SubscriptionMessageType.TS_FILE_HANDLER.getType(), message.getMessageType())) { + SubscriptionMessageType.TS_FILE_HANDLER.getType(), message.getMessageType()) + && + // do not delete file that can resume from breakpoint + !inFlightFilesCommitContextSet.contains(message.getCommitContext())) { try { message.getTsFileHandler().deleteFile(); } catch (final Exception ignored) { @@ -1083,7 +1123,7 @@ protected void nack(final Iterable messages) throws Subscri .computeIfAbsent(message.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) .add(message.getCommitContext()); } - for (final Map.Entry> entry : + for (final Entry> entry : dataNodeIdToSubscriptionCommitContexts.entrySet()) { commitInternal(entry.getKey(), entry.getValue(), true); } @@ -1093,7 +1133,7 @@ private void nack(final List responses) throws Subscri final Map> dataNodeIdToSubscriptionCommitContexts = new HashMap<>(); for (final SubscriptionPollResponse response : responses) { - // the actual file name cannot be obtained here through the response... + // there is no stale intermediate file here dataNodeIdToSubscriptionCommitContexts .computeIfAbsent(response.getCommitContext().getDataNodeId(), (id) -> new ArrayList<>()) .add(response.getCommitContext()); @@ -1338,7 +1378,6 @@ public abstract static class Builder { protected boolean fileSaveFsync = ConsumerConstant.FILE_SAVE_FSYNC_DEFAULT_VALUE; protected int thriftMaxFrameSize = SessionConfig.DEFAULT_MAX_FRAME_SIZE; - protected int maxPollParallelism = ConsumerConstant.MAX_POLL_PARALLELISM_DEFAULT_VALUE; public Builder host(final String host) { @@ -1424,6 +1463,7 @@ protected Map coreReportMessage() { result.put("consumerGroupId", consumerGroupId); result.put("isClosed", isClosed.toString()); result.put("fileSaveDir", fileSaveDir); + result.put("inFlightFilesCommitContextSet", inFlightFilesCommitContextSet.toString()); result.put("subscribedTopicNames", subscribedTopics.keySet().toString()); return result; } @@ -1439,6 +1479,7 @@ protected Map allReportMessage() { result.put("isReleased", isReleased.toString()); result.put("fileSaveDir", fileSaveDir); result.put("fileSaveFsync", String.valueOf(fileSaveFsync)); + result.put("inFlightFilesCommitContextSet", inFlightFilesCommitContextSet.toString()); result.put("thriftMaxFrameSize", String.valueOf(thriftMaxFrameSize)); result.put("maxPollParallelism", String.valueOf(maxPollParallelism)); result.put("subscribedTopics", subscribedTopics.toString()); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java index 345886542423..b8f35b392b0c 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionExecutorServiceManager.java @@ -31,7 +31,6 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; public final class SubscriptionExecutorServiceManager { @@ -285,10 +284,12 @@ int getAvailableCount() { if (!isShutdown()) { synchronized (this) { if (!isShutdown()) { - return Math.max( - ((ThreadPoolExecutor) this.executor).getPoolSize() - - ((ThreadPoolExecutor) this.executor).getActiveCount(), - 0); + // TODO: temporarily disable multiple poll + return 0; + // return Math.max( + // ((ThreadPoolExecutor) this.executor).getCorePoolSize() + // - ((ThreadPoolExecutor) this.executor).getActiveCount(), + // 0); } } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java index ef03b6f20cbd..dc717e6a4c6b 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java @@ -299,34 +299,35 @@ Map unsubscribe(final Set topicNames) throws Subscr return unsubscribeResp.getTopics(); } - List poll(final Set topicNames) throws SubscriptionException { + List poll(final Set topicNames, final long timeoutMs) + throws SubscriptionException { return poll( new SubscriptionPollRequest( SubscriptionPollRequestType.POLL.getType(), new PollPayload(topicNames), - 0L, + timeoutMs, thriftMaxFrameSize)); } List pollFile( - final SubscriptionCommitContext commitContext, final long writingOffset) + final SubscriptionCommitContext commitContext, final long writingOffset, final long timeoutMs) throws SubscriptionException { return poll( new SubscriptionPollRequest( SubscriptionPollRequestType.POLL_FILE.getType(), new PollFilePayload(commitContext, writingOffset), - 0L, + timeoutMs, thriftMaxFrameSize)); } List pollTablets( - final SubscriptionCommitContext commitContext, final int offset) + final SubscriptionCommitContext commitContext, final int offset, final long timeoutMs) throws SubscriptionException { return poll( new SubscriptionPollRequest( SubscriptionPollRequestType.POLL_TABLETS.getType(), new PollTabletsPayload(commitContext, offset), - 0L, + timeoutMs, thriftMaxFrameSize)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java index 7f29b3c55f99..e5383bfaae41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java @@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory; import java.util.Objects; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicReference; public class PipeProcessorSubtask extends PipeReportableSubtask { @@ -96,7 +96,7 @@ public PipeProcessorSubtask( @Override public void bindExecutors( final ListeningExecutorService subtaskWorkerThreadPoolExecutor, - final ExecutorService ignored, + final ScheduledExecutorService ignored, final PipeSubtaskScheduler subtaskScheduler) { this.subtaskWorkerThreadPoolExecutor = subtaskWorkerThreadPoolExecutor; this.subtaskScheduler = subtaskScheduler; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java index c7b8e1f0615b..44374adb22a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java @@ -25,7 +25,6 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; -import org.apache.tsfile.exception.write.WriteProcessException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +57,7 @@ protected PipeTabletEventBatch(final int maxDelayInMs) { * @return {@code true} if the batch can be transferred */ public synchronized boolean onEvent(final TabletInsertionEvent event) - throws WALPipeException, IOException, WriteProcessException { + throws WALPipeException, IOException { if (isClosed || !(event instanceof EnrichedEvent)) { return false; } @@ -94,7 +93,7 @@ public synchronized boolean onEvent(final TabletInsertionEvent event) * exceptions and do not return {@code false} here. */ protected abstract boolean constructBatch(final TabletInsertionEvent event) - throws WALPipeException, IOException, WriteProcessException; + throws WALPipeException, IOException; public boolean shouldEmit() { return totalBufferSize >= getMaxBatchSizeInBytes() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 33b4a910e4ac..77e091ddd6b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -475,14 +475,19 @@ public boolean isTableModelEvent() { /////////////////////////// TsFileInsertionEvent /////////////////////////// @Override - public Iterable toTabletInsertionEvents() { + public Iterable toTabletInsertionEvents() throws PipeException { + return toTabletInsertionEvents(Long.MAX_VALUE); + } + + public Iterable toTabletInsertionEvents(final long timeoutMs) + throws PipeException { try { if (!waitForTsFileClose()) { LOGGER.warn( "Pipe skipping temporary TsFile's parsing which shouldn't be transferred: {}", tsFile); return Collections.emptyList(); } - waitForResourceEnough4Parsing(); + waitForResourceEnough4Parsing(timeoutMs); return initEventParser().toTabletInsertionEvents(); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -496,31 +501,49 @@ public Iterable toTabletInsertionEvents() { } } - private void waitForResourceEnough4Parsing() throws InterruptedException { + private void waitForResourceEnough4Parsing(final long timeoutMs) throws InterruptedException { final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory(); if (memoryManager.isEnough4TabletParsing()) { return; } + final long startTime = System.currentTimeMillis(); + long lastRecordTime = startTime; + final long memoryCheckIntervalMs = PipeConfig.getInstance().getPipeTsFileParserCheckMemoryEnoughIntervalMs(); - final long startTime = System.currentTimeMillis(); while (!memoryManager.isEnough4TabletParsing()) { Thread.sleep(memoryCheckIntervalMs); - } - final double waitTimeSeconds = (System.currentTimeMillis() - startTime) / 1000.0; - if (waitTimeSeconds > 1.0) { - LOGGER.info( - "Wait for resource enough for parsing {} for {} seconds.", - resource != null ? resource.getTsFilePath() : "tsfile", - waitTimeSeconds); - } else if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Wait for resource enough for parsing {} for {} seconds.", - resource != null ? resource.getTsFilePath() : "tsfile", - waitTimeSeconds); + final long currentTime = System.currentTimeMillis(); + final double elapsedRecordTimeSeconds = (currentTime - lastRecordTime) / 1000.0; + final double waitTimeSeconds = (currentTime - startTime) / 1000.0; + if (elapsedRecordTimeSeconds > 10.0) { + LOGGER.info( + "Wait for resource enough for parsing {} for {} seconds.", + resource != null ? resource.getTsFilePath() : "tsfile", + waitTimeSeconds); + lastRecordTime = currentTime; + } else if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Wait for resource enough for parsing {} for {} seconds.", + resource != null ? resource.getTsFilePath() : "tsfile", + waitTimeSeconds); + } + + if (waitTimeSeconds * 1000 > timeoutMs) { + // should contain 'TimeoutException' in exception message + throw new InterruptedException( + String.format("TimeoutException: Waited %s seconds", waitTimeSeconds)); + } } + + final long currentTime = System.currentTimeMillis(); + final double waitTimeSeconds = (currentTime - startTime) / 1000.0; + LOGGER.info( + "Wait for resource enough for parsing {} for {} seconds.", + resource != null ? resource.getTsFilePath() : "tsfile", + waitTimeSeconds); } /** The method is used to prevent circular replication in PipeConsensus */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index f0caf0a631b8..8ae6235099c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -59,6 +59,7 @@ public class PipeMemoryManager { PipeConfig.getInstance().getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold(); private volatile long usedMemorySizeInBytesOfTablets; + // Used to control the memory allocated for managing slice tsfile in subscription module. private static final double TS_FILE_MEMORY_REJECT_THRESHOLD = PipeConfig.getInstance().getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold(); private volatile long usedMemorySizeInBytesOfTsFiles; @@ -78,6 +79,11 @@ public boolean isEnough4TabletParsing() { < 0.95 * TABLET_MEMORY_REJECT_THRESHOLD * TOTAL_MEMORY_SIZE_IN_BYTES; } + public boolean isEnough4TsFileSlicing() { + return (double) usedMemorySizeInBytesOfTsFiles + < 0.95 * TS_FILE_MEMORY_REJECT_THRESHOLD * TOTAL_MEMORY_SIZE_IN_BYTES; + } + public synchronized PipeMemoryBlock forceAllocate(long sizeInBytes) throws PipeRuntimeOutOfMemoryCriticalException { return forceAllocate(sizeInBytes, PipeMemoryBlockType.NORMAL); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java index 71f319374cd5..0eb32e21f5d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.subscription.agent; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiver; import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiverV1; import org.apache.iotdb.rpc.RpcUtils; @@ -69,6 +70,18 @@ public TPipeSubscribeResp handle(final TPipeSubscribeReq req) { } } + public long remainingMs() { + return remainingMs(PipeSubscribeRequestVersion.VERSION_1.getVersion()); // default to VERSION_1 + } + + public long remainingMs(final byte reqVersion) { + if (RECEIVER_CONSTRUCTORS.containsKey(reqVersion)) { + return getReceiver(reqVersion).remainingMs(); + } else { + return SubscriptionConfig.getInstance().getSubscriptionDefaultTimeoutInMs(); + } + } + private SubscriptionReceiver getReceiver(final byte reqVersion) { if (receiverThreadLocal.get() == null) { return setAndGetReceiver(reqVersion); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index 105ed26082e5..433052409d9f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -23,11 +23,14 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.agent.task.execution.PipeSubtaskExecutorManager; import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.batch.SubscriptionPipeEventBatches; +import org.apache.iotdb.db.subscription.task.subtask.SubscriptionReceiverSubtask; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; @@ -160,6 +163,15 @@ protected void releaseWriteLock() { lock.writeLock().unlock(); } + /////////////////////////////// subtask /////////////////////////////// + + protected void executeReceiverSubtask( + final SubscriptionReceiverSubtask subtask, final long timeoutMs) throws Exception { + PipeSubtaskExecutorManager.getInstance() + .getSubscriptionExecutor() + .executeReceiverSubtask(subtask, timeoutMs); + } + /////////////////////////////// poll /////////////////////////////// public SubscriptionEvent poll(final String consumerId) { @@ -176,7 +188,15 @@ public SubscriptionEvent pollInternal(final String consumerId) { if (prefetchingQueue.isEmpty()) { states.markMissingPrefetch(); - tryPrefetch(); + try { + executeReceiverSubtask( + () -> { + tryPrefetch(); + return null; + }, + SubscriptionAgent.receiver().remainingMs()); + } catch (final Exception ignored) { + } } if (prefetchingQueue.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java index 09f41cf23cb4..e729fc740951 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; @@ -145,14 +146,23 @@ public SubscriptionEvent pollTablets( // 3. Poll next tablets try { - ev.fetchNextResponse(); - } catch (final Exception ignored) { - // no exceptions will be thrown + executeReceiverSubtask( + () -> { + ev.fetchNextResponse(offset); + return null; + }, + SubscriptionAgent.receiver().remainingMs()); + ev.recordLastPolledTimestamp(); + eventRef.set(ev); + } catch (final Exception e) { + final String errorMessage = + String.format( + "exception occurred when fetching next response: %s, consumer id: %s, commit context: %s, offset: %s, prefetching queue: %s", + e, consumerId, commitContext, offset, this); + LOGGER.warn(errorMessage); + eventRef.set(generateSubscriptionPollErrorResponse(errorMessage)); } - ev.recordLastPolledTimestamp(); - eventRef.set(ev); - return ev; }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java index 5d12eb490ccb..5a7b2f7f31c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.pipe.SubscriptionPipeTsFilePlainEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; @@ -143,16 +144,7 @@ public SubscriptionEvent pollTsFile( eventRef.set(generateSubscriptionPollErrorResponse(errorMessage)); return ev; } - // check offset - if (writingOffset != 0) { - final String errorMessage = - String.format( - "inconsistent offset, current: %s, incoming: %s, consumer: %s, file name: %s, prefetching queue: %s", - 0, writingOffset, consumerId, fileName, this); - LOGGER.warn(errorMessage); - eventRef.set(generateSubscriptionPollErrorResponse(errorMessage)); - return ev; - } + // no need to check offset for resume from breakpoint break; case FILE_PIECE: // check file name @@ -206,26 +198,23 @@ public SubscriptionEvent pollTsFile( // 3. Poll tsfile piece or tsfile seal try { - ev.fetchNextResponse(); + executeReceiverSubtask( + () -> { + ev.fetchNextResponse(writingOffset); + return null; + }, + SubscriptionAgent.receiver().remainingMs()); + ev.recordLastPolledTimestamp(); + eventRef.set(ev); } catch (final Exception e) { - LOGGER.warn( - "Exception occurred when SubscriptionPrefetchingTsFileQueue {} transferring file (with event {}) to consumer {}", - this, - ev, - consumerId, - e); final String errorMessage = String.format( - "Exception occurred when SubscriptionPrefetchingTsFileQueue %s transferring file (with event %s) to consumer %s: %s", - this, ev, consumerId, e); + "exception occurred when fetching next response: %s, consumer id: %s, commit context: %s, writing offset: %s, prefetching queue: %s", + e, consumerId, commitContext, writingOffset, this); LOGGER.warn(errorMessage); eventRef.set(generateSubscriptionPollErrorResponse(errorMessage)); - return ev; } - ev.recordLastPolledTimestamp(); - eventRef.set(ev); - return ev; }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java index bd71db5991cb..ec0ed1fc079f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java @@ -220,12 +220,12 @@ public String getLastPolledConsumerId() { //////////////////////////// prefetch & fetch //////////////////////////// - public void prefetchRemainingResponses() throws IOException { + public void prefetchRemainingResponses() throws Exception { response.prefetchRemainingResponses(); } - public void fetchNextResponse() throws IOException { - response.fetchNextResponse(); + public void fetchNextResponse(final long offset) throws Exception { + response.fetchNextResponse(offset); } //////////////////////////// byte buffer //////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java index a396ff8ee2a0..dbc06881cae0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeEventBatch.java @@ -94,7 +94,7 @@ protected synchronized boolean onEvent( final @NonNull EnrichedEvent event, final Consumer consumer) throws Exception { if (event instanceof TabletInsertionEvent) { - onTabletInsertionEvent((TabletInsertionEvent) event); // no exceptions will be thrown + onTabletInsertionEvent((TabletInsertionEvent) event); enrichedEvents.add(event); } else if (event instanceof TsFileInsertionEvent) { onTsFileInsertionEvent((TsFileInsertionEvent) event); @@ -108,9 +108,9 @@ protected synchronized boolean onEvent( /////////////////////////////// utility /////////////////////////////// - protected abstract void onTabletInsertionEvent(final TabletInsertionEvent event) throws Exception; + protected abstract void onTabletInsertionEvent(final TabletInsertionEvent event); - protected abstract void onTsFileInsertionEvent(final TsFileInsertionEvent event) throws Exception; + protected abstract void onTsFileInsertionEvent(final TsFileInsertionEvent event); protected abstract boolean shouldEmit(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java index b54607c868b4..2100d3839dea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java @@ -22,7 +22,9 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTabletQueue; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; @@ -112,7 +114,9 @@ protected void onTabletInsertionEvent(final TabletInsertionEvent event) { @Override protected void onTsFileInsertionEvent(final TsFileInsertionEvent event) { - for (final TabletInsertionEvent tabletInsertionEvent : event.toTabletInsertionEvents()) { + for (final TabletInsertionEvent tabletInsertionEvent : + ((PipeTsFileInsertionEvent) event) + .toTabletInsertionEvents(SubscriptionAgent.receiver().remainingMs())) { onTabletInsertionEvent(tabletInsertionEvent); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java index dc96fc476daa..514795db23a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java @@ -68,8 +68,12 @@ public synchronized void cleanUp() { /////////////////////////////// utility /////////////////////////////// @Override - protected void onTabletInsertionEvent(final TabletInsertionEvent event) throws Exception { - batch.onEvent(event); // no exceptions will be thrown + protected void onTabletInsertionEvent(final TabletInsertionEvent event) { + try { + batch.onEvent(event); + } catch (final Exception ignored) { + // no exceptions will be thrown + } ((EnrichedEvent) event) .decreaseReferenceCount( SubscriptionPipeTsFileEventBatch.class.getName(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventExtendableResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventExtendableResponse.java index 07dfaabdf4ff..dbbe9d898c87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventExtendableResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventExtendableResponse.java @@ -57,16 +57,6 @@ public CachedSubscriptionPollResponse getCurrentResponse() { return peekFirst(); } - @Override - public void fetchNextResponse() throws IOException { - prefetchRemainingResponses(); - if (Objects.isNull(poll())) { - LOGGER.warn( - "SubscriptionEventExtendableResponse {} is empty when fetching next response (broken invariant)", - this); - } - } - @Override public void trySerializeCurrentResponse() { SubscriptionPollResponseCache.getInstance().trySerialize(getCurrentResponse()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventResponse.java index 211f0905afa9..3ded870feedf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventResponse.java @@ -28,9 +28,9 @@ public interface SubscriptionEventResponse { E getCurrentResponse(); - void prefetchRemainingResponses() throws IOException; + void prefetchRemainingResponses() throws Exception; - void fetchNextResponse() throws IOException; + void fetchNextResponse(final long offset) throws Exception; /////////////////////////////// byte buffer /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java index dbc48ebda00c..7c72e65b6d66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java @@ -65,7 +65,7 @@ public void prefetchRemainingResponses() { } @Override - public void fetchNextResponse() { + public void fetchNextResponse(final long offset) { // do nothing } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java index 5ed3f7fd56d4..e2590f5ff322 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java @@ -77,6 +77,16 @@ public void prefetchRemainingResponses() { offer(generateNextTabletResponse()); } + @Override + public void fetchNextResponse(final long offset /* unused */) { + prefetchRemainingResponses(); + if (Objects.isNull(poll())) { + LOGGER.warn( + "SubscriptionEventTabletResponse {} is empty when fetching next response (broken invariant)", + this); + } + } + @Override public synchronized void nack() { if (nextOffset.get() == 1) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java index 63a4c2d1f93a..6c102c21912d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java @@ -19,9 +19,12 @@ package org.apache.iotdb.db.subscription.event.response; +import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; +import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; import org.apache.iotdb.db.pipe.resource.memory.PipeTsFileMemoryBlock; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.event.cache.CachedSubscriptionPollResponse; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload; @@ -67,12 +70,18 @@ public SubscriptionEventTsFileResponse( } @Override - public void prefetchRemainingResponses() throws IOException { - if (hasNoMore) { - return; - } + public void prefetchRemainingResponses() { + // do nothing + } - generateNextTsFileResponse().ifPresent(super::offer); + @Override + public void fetchNextResponse(final long offset) throws Exception { + generateNextTsFileResponse(offset).ifPresent(super::offer); + if (Objects.isNull(poll())) { + LOGGER.warn( + "SubscriptionEventTsFileResponse {} is empty when fetching next response (broken invariant)", + this); + } } @Override @@ -103,8 +112,13 @@ private void init() { commitContext)); } + private synchronized Optional generateNextTsFileResponse( + final long offset) throws SubscriptionException, IOException, InterruptedException { + return Optional.of(generateResponseWithPieceOrSealPayload(offset)); + } + private synchronized Optional generateNextTsFileResponse() - throws IOException { + throws SubscriptionException, IOException, InterruptedException { final SubscriptionPollResponse previousResponse = peekLast(); if (Objects.isNull(previousResponse)) { LOGGER.warn( @@ -137,7 +151,7 @@ private synchronized Optional generateNextTsFile } private @NonNull CachedSubscriptionPollResponse generateResponseWithPieceOrSealPayload( - final long writingOffset) throws IOException { + final long writingOffset) throws SubscriptionException, IOException, InterruptedException { final long tsFileLength = tsFile.length(); if (writingOffset >= tsFileLength) { // generate subscription poll response with seal payload @@ -159,6 +173,7 @@ private synchronized Optional generateNextTsFile bufferSize = readFileBufferSize; } + waitForResourceEnough4Slicing(SubscriptionAgent.receiver().remainingMs()); try (final RandomAccessFile reader = new RandomAccessFile(tsFile, "r")) { reader.seek(writingOffset); @@ -185,4 +200,48 @@ private synchronized Optional generateNextTsFile return response; } } + + private void waitForResourceEnough4Slicing(final long timeoutMs) throws InterruptedException { + final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory(); + if (memoryManager.isEnough4TsFileSlicing()) { + return; + } + + final long startTime = System.currentTimeMillis(); + long lastRecordTime = startTime; + + final long memoryCheckIntervalMs = + PipeConfig.getInstance().getPipeTsFileParserCheckMemoryEnoughIntervalMs(); + while (!memoryManager.isEnough4TsFileSlicing()) { + Thread.sleep(memoryCheckIntervalMs); + + final long currentTime = System.currentTimeMillis(); + final double elapsedRecordTimeSeconds = (currentTime - lastRecordTime) / 1000.0; + final double waitTimeSeconds = (currentTime - startTime) / 1000.0; + if (elapsedRecordTimeSeconds > 10.0) { + LOGGER.info( + "Wait for resource enough for slicing tsfile {} for {} seconds.", + tsFile, + waitTimeSeconds); + lastRecordTime = currentTime; + } else if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Wait for resource enough for slicing tsfile {} for {} seconds.", + tsFile, + waitTimeSeconds); + } + + if (waitTimeSeconds * 1000 > timeoutMs) { + // should contain 'TimeoutException' in exception message + // see org.apache.iotdb.rpc.subscription.exception.SubscriptionTimeoutException.KEYWORD + throw new InterruptedException( + String.format("TimeoutException: Waited %s seconds", waitTimeSeconds)); + } + } + + final long currentTime = System.currentTimeMillis(); + final double waitTimeSeconds = (currentTime - startTime) / 1000.0; + LOGGER.info( + "Wait for resource enough for slicing tsfile {} for {} seconds.", tsFile, waitTimeSeconds); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java index 5d2d4b050b14..c636e1b2dc6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java @@ -30,4 +30,6 @@ public interface SubscriptionReceiver { PipeSubscribeRequestVersion getVersion(); void handleExit(); + + long remainingMs(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 0a03f28bf7d3..73ee10c28544 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; @@ -68,6 +69,7 @@ import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeUnsubscribeResp; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeReq; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.iotdb.session.subscription.util.PollTimer; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -101,23 +103,7 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { PipeSubscribeResponseType.ACK.getType()); private final ThreadLocal consumerConfigThreadLocal = new ThreadLocal<>(); - - @Override - public PipeSubscribeRequestVersion getVersion() { - return PipeSubscribeRequestVersion.VERSION_1; - } - - @Override - public void handleExit() { - final ConsumerConfig consumerConfig = consumerConfigThreadLocal.get(); - if (Objects.nonNull(consumerConfig)) { - LOGGER.info( - "Subscription: remove consumer config {} when handling exit", - consumerConfigThreadLocal.get()); - // closeConsumer(consumerConfig); - consumerConfigThreadLocal.remove(); - } - } + private final ThreadLocal pollTimerThreadLocal = new ThreadLocal<>(); @Override public final TPipeSubscribeResp handle(final TPipeSubscribeReq req) { @@ -155,6 +141,32 @@ public final TPipeSubscribeResp handle(final TPipeSubscribeReq req) { PipeSubscribeResponseType.ACK.getType()); } + @Override + public PipeSubscribeRequestVersion getVersion() { + return PipeSubscribeRequestVersion.VERSION_1; + } + + @Override + public void handleExit() { + final ConsumerConfig consumerConfig = consumerConfigThreadLocal.get(); + if (Objects.nonNull(consumerConfig)) { + LOGGER.info( + "Subscription: remove consumer config {} when handling exit", + consumerConfigThreadLocal.get()); + // closeConsumer(consumerConfig); + consumerConfigThreadLocal.remove(); + } + } + + @Override + public long remainingMs() { + final PollTimer pollTimer = pollTimerThreadLocal.get(); + if (Objects.isNull(pollTimer)) { + return SubscriptionConfig.getInstance().getSubscriptionDefaultTimeoutInMs(); + } + return pollTimer.remainingMs(); + } + private TPipeSubscribeResp handlePipeSubscribeHandshake(final PipeSubscribeHandshakeReq req) { try { return handlePipeSubscribeHandshakeInternal(req); @@ -342,6 +354,24 @@ private TPipeSubscribeResp handlePipeSubscribeUnsubscribeInternal( } private TPipeSubscribeResp handlePipeSubscribePoll(final PipeSubscribePollReq req) { + try { + return handlePipeSubscribePollInternal(req); + } catch (final Exception e) { + LOGGER.warn("Exception occurred when polling with request {}", req, e); + final String exceptionMessage = + String.format( + "Subscription: something unexpected happened when polling with request %s: %s", + req, e); + return PipeSubscribePollResp.toTPipeSubscribeResp( + RpcUtils.getStatus(TSStatusCode.SUBSCRIPTION_POLL_ERROR, exceptionMessage), + Collections.emptyList()); + } finally { + pollTimerThreadLocal.remove(); + } + } + + private TPipeSubscribeResp handlePipeSubscribePollInternal(final PipeSubscribePollReq req) + throws SubscriptionException { final ConsumerConfig consumerConfig = consumerConfigThreadLocal.get(); if (Objects.isNull(consumerConfig)) { LOGGER.warn( @@ -351,120 +381,113 @@ private TPipeSubscribeResp handlePipeSubscribePoll(final PipeSubscribePollReq re final List events; final SubscriptionPollRequest request = req.getRequest(); + + pollTimerThreadLocal.set(new PollTimer(System.currentTimeMillis(), request.getTimeoutMs())); + final long maxBytes = (long) (request.getMaxBytes() * POLL_PAYLOAD_SIZE_EXCEED_THRESHOLD); - try { - final short requestType = request.getRequestType(); - if (SubscriptionPollRequestType.isValidatedRequestType(requestType)) { - switch (SubscriptionPollRequestType.valueOf(requestType)) { - case POLL: - events = - handlePipeSubscribePollInternal( - consumerConfig, (PollPayload) request.getPayload(), maxBytes); - break; - case POLL_FILE: - events = - handlePipeSubscribePollTsFileInternal( - consumerConfig, (PollFilePayload) request.getPayload()); - break; - case POLL_TABLETS: - events = - handlePipeSubscribePollTabletsInternal( - consumerConfig, (PollTabletsPayload) request.getPayload()); - break; - default: - events = null; - break; - } - } else { - events = null; - } - if (Objects.isNull(events)) { - throw new SubscriptionException(String.format("unexpected request type: %s", requestType)); + final short requestType = request.getRequestType(); + if (SubscriptionPollRequestType.isValidatedRequestType(requestType)) { + switch (SubscriptionPollRequestType.valueOf(requestType)) { + case POLL: + events = + handlePipeSubscribePollRequest( + consumerConfig, (PollPayload) request.getPayload(), maxBytes); + break; + case POLL_FILE: + events = + handlePipeSubscribePollTsFileRequest( + consumerConfig, (PollFilePayload) request.getPayload()); + break; + case POLL_TABLETS: + events = + handlePipeSubscribePollTabletsRequest( + consumerConfig, (PollTabletsPayload) request.getPayload()); + break; + default: + events = null; + break; } + } else { + events = null; + } - // generate response - final AtomicLong totalSize = new AtomicLong(); - return PipeSubscribePollResp.toTPipeSubscribeResp( - RpcUtils.SUCCESS_STATUS, - events.stream() - .map( - (event) -> { - final SubscriptionCommitContext commitContext = event.getCommitContext(); - final SubscriptionPollResponse response = event.getCurrentResponse(); - if (Objects.isNull(response)) { - LOGGER.warn( - "Subscription: consumer {} poll null response for event {} with request: {}", - consumerConfig, - event, - req.getRequest()); - // nack - SubscriptionAgent.broker() - .commit(consumerConfig, Collections.singletonList(commitContext), true); - return null; - } + if (Objects.isNull(events)) { + throw new SubscriptionException(String.format("unexpected request type: %s", requestType)); + } - try { - final ByteBuffer byteBuffer = event.getCurrentResponseByteBuffer(); - - // payload size control - final long size = event.getCurrentResponseSize(); - if (totalSize.get() + size > maxBytes) { - throw new SubscriptionPayloadExceedException( - String.format( - "payload size %s byte(s) will exceed the threshold %s byte(s)", - totalSize.get() + size, maxBytes)); - } - totalSize.getAndAdd(size); - - SubscriptionPrefetchingQueueMetrics.getInstance() - .mark( - SubscriptionPrefetchingQueue.generatePrefetchingQueueId( - commitContext.getConsumerGroupId(), commitContext.getTopicName()), - size); - event.invalidateCurrentResponseByteBuffer(); - LOGGER.info( - "Subscription: consumer {} poll {} successfully with request: {}", + // generate response + final AtomicLong totalSize = new AtomicLong(); + return PipeSubscribePollResp.toTPipeSubscribeResp( + RpcUtils.SUCCESS_STATUS, + events.stream() + .map( + (event) -> { + final SubscriptionCommitContext commitContext = event.getCommitContext(); + final SubscriptionPollResponse response = event.getCurrentResponse(); + if (Objects.isNull(response)) { + LOGGER.warn( + "Subscription: consumer {} poll null response for event {} with request: {}", + consumerConfig, + event, + req.getRequest()); + // nack + SubscriptionAgent.broker() + .commit(consumerConfig, Collections.singletonList(commitContext), true); + return null; + } + + try { + final ByteBuffer byteBuffer = event.getCurrentResponseByteBuffer(); + + // payload size control + final long size = event.getCurrentResponseSize(); + if (totalSize.get() + size > maxBytes) { + throw new SubscriptionPayloadExceedException( + String.format( + "payload size %s byte(s) will exceed the threshold %s byte(s)", + totalSize.get() + size, maxBytes)); + } + totalSize.getAndAdd(size); + + SubscriptionPrefetchingQueueMetrics.getInstance() + .mark( + SubscriptionPrefetchingQueue.generatePrefetchingQueueId( + commitContext.getConsumerGroupId(), commitContext.getTopicName()), + size); + event.invalidateCurrentResponseByteBuffer(); + LOGGER.info( + "Subscription: consumer {} poll {} successfully with request: {}", + consumerConfig, + response, + req.getRequest()); + return byteBuffer; + } catch (final Exception e) { + if (e instanceof SubscriptionPayloadExceedException) { + LOGGER.error( + "Subscription: consumer {} poll excessive payload {} with request: {}, something unexpected happened with parameter configuration or payload control...", + consumerConfig, + response, + req.getRequest(), + e); + } else { + LOGGER.warn( + "Subscription: consumer {} poll {} failed with request: {}", consumerConfig, response, - req.getRequest()); - return byteBuffer; - } catch (final Exception e) { - if (e instanceof SubscriptionPayloadExceedException) { - LOGGER.error( - "Subscription: consumer {} poll excessive payload {} with request: {}, something unexpected happened with parameter configuration or payload control...", - consumerConfig, - response, - req.getRequest(), - e); - } else { - LOGGER.warn( - "Subscription: consumer {} poll {} failed with request: {}", - consumerConfig, - response, - req.getRequest(), - e); - } - // nack - SubscriptionAgent.broker() - .commit(consumerConfig, Collections.singletonList(commitContext), true); - return null; + req.getRequest(), + e); } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList())); - } catch (final Exception e) { - LOGGER.warn("Exception occurred when polling with request {}", req, e); - final String exceptionMessage = - String.format( - "Subscription: something unexpected happened when polling with request %s: %s", - req, e); - return PipeSubscribePollResp.toTPipeSubscribeResp( - RpcUtils.getStatus(TSStatusCode.SUBSCRIPTION_POLL_ERROR, exceptionMessage), - Collections.emptyList()); - } + // nack + SubscriptionAgent.broker() + .commit(consumerConfig, Collections.singletonList(commitContext), true); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())); } - private List handlePipeSubscribePollInternal( + private List handlePipeSubscribePollRequest( final ConsumerConfig consumerConfig, final PollPayload messagePayload, final long maxBytes) { final Set subscribedTopicNames = SubscriptionAgent.consumer() @@ -480,14 +503,14 @@ private List handlePipeSubscribePollInternal( return SubscriptionAgent.broker().poll(consumerConfig, topicNames, maxBytes); } - private List handlePipeSubscribePollTsFileInternal( + private List handlePipeSubscribePollTsFileRequest( final ConsumerConfig consumerConfig, final PollFilePayload messagePayload) { return SubscriptionAgent.broker() .pollTsFile( consumerConfig, messagePayload.getCommitContext(), messagePayload.getWritingOffset()); } - private List handlePipeSubscribePollTabletsInternal( + private List handlePipeSubscribePollTabletsRequest( final ConsumerConfig consumerConfig, final PollTabletsPayload messagePayload) { return SubscriptionAgent.broker() .pollTablets(consumerConfig, messagePayload.getCommitContext(), messagePayload.getOffset()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java index 76b0a7db2f6c..f473a56a4478 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java @@ -20,14 +20,64 @@ package org.apache.iotdb.db.subscription.task.execution; import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.pipe.agent.task.execution.PipeSubtaskExecutor; +import org.apache.iotdb.commons.pipe.agent.task.execution.PipeSubtaskScheduler; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor; +import org.apache.iotdb.db.subscription.task.subtask.SubscriptionReceiverSubtask; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; public class SubscriptionSubtaskExecutor extends PipeConnectorSubtaskExecutor { + private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionSubtaskExecutor.class); + + private final AtomicLong submittedReceiverSubtasks = new AtomicLong(0); + public SubscriptionSubtaskExecutor() { super( SubscriptionConfig.getInstance().getSubscriptionSubtaskExecutorMaxThreadNum(), ThreadName.SUBSCRIPTION_EXECUTOR_POOL); } + + @Override + protected PipeSubtaskScheduler schedulerSupplier(final PipeSubtaskExecutor executor) { + return new SubscriptionSubtaskScheduler((SubscriptionSubtaskExecutor) executor); + } + + public void executeReceiverSubtask( + final SubscriptionReceiverSubtask subtask, final long timeoutMs) throws Exception { + if (!super.hasAvailableThread()) { + subtask.call(); // non-strict timeout + return; + } + + submittedReceiverSubtasks.incrementAndGet(); + try { + final Future future = subtaskWorkerThreadPoolExecutor.submit(subtask); + try { + future.get(timeoutMs, TimeUnit.MILLISECONDS); // strict timeout + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // restore interrupted state + future.cancel(true); + throw e; + } catch (final ExecutionException | TimeoutException e) { + future.cancel(true); + throw e; + } + } finally { + submittedReceiverSubtasks.decrementAndGet(); + } + } + + public boolean hasSubmittedReceiverSubtasks() { + return submittedReceiverSubtasks.get() > 0; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskScheduler.java new file mode 100644 index 000000000000..44072d1d8237 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskScheduler.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.task.execution; + +import org.apache.iotdb.commons.pipe.agent.task.execution.PipeSubtaskScheduler; + +public class SubscriptionSubtaskScheduler extends PipeSubtaskScheduler { + + private final SubscriptionSubtaskExecutor executor; + + public SubscriptionSubtaskScheduler(final SubscriptionSubtaskExecutor executor) { + super(executor); + + this.executor = executor; + } + + @Override + public boolean schedule() { + // prioritize executing SubscriptionReceiverSubtask over SubscriptionConnectorSubtask + return !executor.hasSubmittedReceiverSubtasks() && super.schedule(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java index 1b4c04251591..c5b841a8c66f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java @@ -20,14 +20,19 @@ package org.apache.iotdb.db.subscription.task.subtask; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtask; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.event.Event; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; + public class SubscriptionConnectorSubtask extends PipeConnectorSubtask { private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionConnectorSubtask.class); @@ -55,15 +60,6 @@ public SubscriptionConnectorSubtask( this.consumerGroupId = consumerGroupId; } - @Override - protected boolean executeOnce() { - if (isClosed.get()) { - return false; - } - - return SubscriptionAgent.broker().executePrefetch(consumerGroupId, topicName); - } - public String getTopicName() { return topicName; } @@ -76,6 +72,36 @@ public UnboundedBlockingPendingQueue getInputPendingQueue() { return inputPendingQueue; } + //////////////////////////// execution & callback //////////////////////////// + + @Override + protected void registerCallbackHookAfterSubmit(final ListenableFuture future) { + final ListenableFuture nextFuture = + Futures.withTimeout( + future, + Duration.ofSeconds( + SubscriptionConfig.getInstance().getSubscriptionDefaultTimeoutInMs()), + subtaskCallbackListeningExecutor); + Futures.addCallback(nextFuture, this, subtaskCallbackListeningExecutor); + } + + @Override + public synchronized void onFailure(final Throwable throwable) { + isSubmitted = false; + + // just resubmit + submitSelf(); + } + + @Override + protected boolean executeOnce() { + if (isClosed.get()) { + return false; + } + + return SubscriptionAgent.broker().executePrefetch(consumerGroupId, topicName); + } + //////////////////////////// APIs provided for metric framework //////////////////////////// @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionReceiverSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionReceiverSubtask.java new file mode 100644 index 000000000000..a070e6dbe6be --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionReceiverSubtask.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.task.subtask; + +import java.util.concurrent.Callable; + +public interface SubscriptionReceiverSubtask extends Callable {} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 59e20daae55d..0a7dae166dd6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -288,12 +288,13 @@ public class CommonConfig { private int subscriptionPrefetchTsFileBatchMaxDelayInMs = 5000; // 5s private long subscriptionPrefetchTsFileBatchMaxSizeInBytes = 80 * MB; private int subscriptionPollMaxBlockingTimeMs = 500; - private int subscriptionSerializeMaxBlockingTimeMs = 100; + private int subscriptionDefaultTimeoutInMs = 10_000; // 10s private long subscriptionLaunchRetryIntervalMs = 1000; private int subscriptionRecycleUncommittedEventIntervalMs = 600000; // 600s private long subscriptionReadFileBufferSize = 8 * MB; private long subscriptionReadTabletBufferSize = 8 * MB; private long subscriptionTsFileDeduplicationWindowSeconds = 120; // 120s + private volatile long subscriptionTsFileSlicerCheckMemoryEnoughIntervalMs = 10L; private long subscriptionMetaSyncerInitialSyncDelayMinutes = 3; private long subscriptionMetaSyncerSyncIntervalMinutes = 3; @@ -1295,13 +1296,12 @@ public void setSubscriptionPollMaxBlockingTimeMs(int subscriptionPollMaxBlocking this.subscriptionPollMaxBlockingTimeMs = subscriptionPollMaxBlockingTimeMs; } - public int getSubscriptionSerializeMaxBlockingTimeMs() { - return subscriptionSerializeMaxBlockingTimeMs; + public int getSubscriptionDefaultTimeoutInMs() { + return subscriptionDefaultTimeoutInMs; } - public void setSubscriptionSerializeMaxBlockingTimeMs( - int subscriptionSerializeMaxBlockingTimeMs) { - this.subscriptionSerializeMaxBlockingTimeMs = subscriptionSerializeMaxBlockingTimeMs; + public void setSubscriptionDefaultTimeoutInMs(final int subscriptionDefaultTimeoutInMs) { + this.subscriptionDefaultTimeoutInMs = subscriptionDefaultTimeoutInMs; } public long getSubscriptionLaunchRetryIntervalMs() { @@ -1348,6 +1348,16 @@ public void setSubscriptionTsFileDeduplicationWindowSeconds( subscriptionTsFileDeduplicationWindowSeconds; } + public long getSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs() { + return subscriptionTsFileSlicerCheckMemoryEnoughIntervalMs; + } + + public void setSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs( + long subscriptionTsFileSlicerCheckMemoryEnoughIntervalMs) { + this.subscriptionTsFileSlicerCheckMemoryEnoughIntervalMs = + subscriptionTsFileSlicerCheckMemoryEnoughIntervalMs; + } + public long getSubscriptionMetaSyncerInitialSyncDelayMinutes() { return subscriptionMetaSyncerInitialSyncDelayMinutes; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 0ecc4387f828..0b175dca2db9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -673,11 +673,11 @@ private void loadSubscriptionProps(Properties properties) { properties.getProperty( "subscription_poll_max_blocking_time_ms", String.valueOf(config.getSubscriptionPollMaxBlockingTimeMs())))); - config.setSubscriptionSerializeMaxBlockingTimeMs( + config.setSubscriptionDefaultTimeoutInMs( Integer.parseInt( properties.getProperty( - "subscription_serialize_max_blocking_time_ms", - String.valueOf(config.getSubscriptionSerializeMaxBlockingTimeMs())))); + "subscription_default_timeout_in_ms", + String.valueOf(config.getSubscriptionDefaultTimeoutInMs())))); config.setSubscriptionLaunchRetryIntervalMs( Long.parseLong( properties.getProperty( @@ -703,6 +703,11 @@ private void loadSubscriptionProps(Properties properties) { properties.getProperty( "subscription_ts_file_deduplication_window_seconds", String.valueOf(config.getSubscriptionTsFileDeduplicationWindowSeconds())))); + config.setSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs( + Long.parseLong( + properties.getProperty( + "subscription_ts_file_slicer_check_memory_enough_interval_ms", + String.valueOf(config.getSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs())))); config.setSubscriptionMetaSyncerInitialSyncDelayMinutes( Long.parseLong( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskExecutor.java index 6130292cdf71..c4c96dad3e77 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskExecutor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskExecutor.java @@ -32,15 +32,17 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; public abstract class PipeSubtaskExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(PipeSubtaskExecutor.class); - private static final ExecutorService subtaskCallbackListeningExecutor = - IoTDBThreadPoolFactory.newSingleThreadExecutor( + private static final ScheduledExecutorService subtaskCallbackListeningExecutor = + IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor( ThreadName.PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL.getName()); + + protected final WrappedThreadPoolExecutor underlyingThreadPool; protected final ListeningExecutorService subtaskWorkerThreadPoolExecutor; private final Map registeredIdSubtaskMapper; @@ -50,13 +52,13 @@ public abstract class PipeSubtaskExecutor { protected PipeSubtaskExecutor( final int corePoolSize, final ThreadName threadName, final boolean disableLogInThreadPool) { - final WrappedThreadPoolExecutor executor = + underlyingThreadPool = (WrappedThreadPoolExecutor) IoTDBThreadPoolFactory.newFixedThreadPool(corePoolSize, threadName.getName()); if (disableLogInThreadPool) { - executor.disableErrorLog(); + underlyingThreadPool.disableErrorLog(); } - subtaskWorkerThreadPoolExecutor = MoreExecutors.listeningDecorator(executor); + subtaskWorkerThreadPoolExecutor = MoreExecutors.listeningDecorator(underlyingThreadPool); registeredIdSubtaskMapper = new ConcurrentHashMap<>(); @@ -74,9 +76,11 @@ public final synchronized void register(final PipeSubtask subtask) { registeredIdSubtaskMapper.put(subtask.getTaskID(), subtask); subtask.bindExecutors( - subtaskWorkerThreadPoolExecutor, - subtaskCallbackListeningExecutor, - new PipeSubtaskScheduler(this)); + subtaskWorkerThreadPoolExecutor, subtaskCallbackListeningExecutor, schedulerSupplier(this)); + } + + protected PipeSubtaskScheduler schedulerSupplier(final PipeSubtaskExecutor executor) { + return new PipeSubtaskScheduler(executor); } public final synchronized void start(final String subTaskID) { @@ -160,4 +164,14 @@ public final int getCorePoolSize() { public final int getRunningSubtaskNumber() { return runningSubtaskNumber; } + + protected final boolean hasAvailableThread() { + // TODO: temporarily disable async receiver subtask execution + return false; + // return getAvailableThreadCount() > 0; + } + + private int getAvailableThreadCount() { + return underlyingThreadPool.getCorePoolSize() - underlyingThreadPool.getActiveCount(); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeAbstractConnectorSubtask.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeAbstractConnectorSubtask.java index f228690a182a..58cc142713d0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeAbstractConnectorSubtask.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeAbstractConnectorSubtask.java @@ -33,7 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; public abstract class PipeAbstractConnectorSubtask extends PipeReportableSubtask { @@ -43,7 +43,7 @@ public abstract class PipeAbstractConnectorSubtask extends PipeReportableSubtask protected PipeConnector outputPipeConnector; // For thread pool to execute callbacks - protected ExecutorService subtaskCallbackListeningExecutor; + protected ScheduledExecutorService subtaskCallbackListeningExecutor; // For controlling subtask submitting, making sure that // a subtask is submitted to only one thread at a time @@ -62,7 +62,7 @@ protected PipeAbstractConnectorSubtask( @Override public void bindExecutors( final ListeningExecutorService subtaskWorkerThreadPoolExecutor, - final ExecutorService subtaskCallbackListeningExecutor, + final ScheduledExecutorService subtaskCallbackListeningExecutor, final PipeSubtaskScheduler subtaskScheduler) { this.subtaskWorkerThreadPoolExecutor = subtaskWorkerThreadPoolExecutor; this.subtaskCallbackListeningExecutor = subtaskCallbackListeningExecutor; @@ -217,10 +217,14 @@ public synchronized void submitSelf() { } final ListenableFuture nextFuture = subtaskWorkerThreadPoolExecutor.submit(this); - Futures.addCallback(nextFuture, this, subtaskCallbackListeningExecutor); + registerCallbackHookAfterSubmit(nextFuture); isSubmitted = true; } + protected void registerCallbackHookAfterSubmit(final ListenableFuture future) { + Futures.addCallback(future, this, subtaskCallbackListeningExecutor); + } + protected synchronized void setLastExceptionEvent(final Event event) { lastExceptionEvent = event; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeSubtask.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeSubtask.java index 2da797c2b3b5..1169711b46d2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeSubtask.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/subtask/PipeSubtask.java @@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -65,7 +65,7 @@ protected PipeSubtask(final String taskID, final long creationTime) { public abstract void bindExecutors( ListeningExecutorService subtaskWorkerThreadPoolExecutor, - ExecutorService subtaskCallbackListeningExecutor, + ScheduledExecutorService subtaskCallbackListeningExecutor, PipeSubtaskScheduler subtaskScheduler); @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/ref/PipePhantomReferenceManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/ref/PipePhantomReferenceManager.java index 47ff0f51ab76..d95616e685d1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/ref/PipePhantomReferenceManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/ref/PipePhantomReferenceManager.java @@ -88,7 +88,9 @@ protected void gcHook() { } else { final long currentPhantomReferenceCount = getPhantomReferenceCount(); if (currentPhantomReferenceCount != lastPhantomReferenceCount) { - LOGGER.info("Remaining pipe phantom reference count: {}", currentPhantomReferenceCount); + if (lastPhantomReferenceCount != -1) { + LOGGER.info("Remaining pipe phantom reference count: {}", currentPhantomReferenceCount); + } lastPhantomReferenceCount = currentPhantomReferenceCount; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/SubscriptionConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/SubscriptionConfig.java index 6a930e29ff6c..fb1f37eae8d2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/SubscriptionConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/SubscriptionConfig.java @@ -57,8 +57,8 @@ public int getSubscriptionPollMaxBlockingTimeMs() { return COMMON_CONFIG.getSubscriptionPollMaxBlockingTimeMs(); } - public int getSubscriptionSerializeMaxBlockingTimeMs() { - return COMMON_CONFIG.getSubscriptionSerializeMaxBlockingTimeMs(); + public int getSubscriptionDefaultTimeoutInMs() { + return COMMON_CONFIG.getSubscriptionDefaultTimeoutInMs(); } public long getSubscriptionLaunchRetryIntervalMs() { @@ -89,6 +89,10 @@ public long getSubscriptionMetaSyncerSyncIntervalMinutes() { return COMMON_CONFIG.getSubscriptionMetaSyncerSyncIntervalMinutes(); } + public long getSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs() { + return COMMON_CONFIG.getSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs(); + } + /////////////////////////////// Utils /////////////////////////////// private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionConfig.class); @@ -113,8 +117,7 @@ public void printAllConfigs() { "SubscriptionPrefetchTsFileBatchMaxSizeInBytes: {}", getSubscriptionPrefetchTsFileBatchMaxSizeInBytes()); LOGGER.info("SubscriptionPollMaxBlockingTimeMs: {}", getSubscriptionPollMaxBlockingTimeMs()); - LOGGER.info( - "SubscriptionSerializeMaxBlockingTimeMs: {}", getSubscriptionSerializeMaxBlockingTimeMs()); + LOGGER.info("SubscriptionDefaultTimeoutInMs: {}", getSubscriptionDefaultTimeoutInMs()); LOGGER.info("SubscriptionLaunchRetryIntervalMs: {}", getSubscriptionLaunchRetryIntervalMs()); LOGGER.info( "SubscriptionRecycleUncommittedEventIntervalMs: {}", @@ -124,6 +127,9 @@ public void printAllConfigs() { LOGGER.info( "SubscriptionTsFileDeduplicationWindowSeconds: {}", getSubscriptionTsFileDeduplicationWindowSeconds()); + LOGGER.info( + "SubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs: {}", + getSubscriptionTsFileSlicerCheckMemoryEnoughIntervalMs()); LOGGER.info( "SubscriptionMetaSyncerInitialSyncDelayMinutes: {}", From 32e701eadd5055e4f3fa01a1f774fa0c9fba8022 Mon Sep 17 00:00:00 2001 From: nanxiang xia <162968176+XNX02@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:51:50 +0800 Subject: [PATCH 5/6] Pipe: Pin unpin log frequency decrease (#14049) Co-authored-by: nanxiang xia <2250337901@qq.com> --- .../resource/tsfile/PipeTsFileResourceManager.java | 14 +++++++------- .../pipe/resource/wal/PipeWALResourceManager.java | 14 ++++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java index d3482653f85c..01bf9f8ca0ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java @@ -80,7 +80,7 @@ private void ttlCheck() throws InterruptedException { PipeConfig.getInstance().getPipeTsFilePinMaxLogNumPerRound(), PipeConfig.getInstance().getPipeTsFilePinMaxLogIntervalRounds(), hardlinkOrCopiedFileToPipeTsFileResourceMap.size()); - + final StringBuilder logBuilder = new StringBuilder(); while (iterator.hasNext()) { final Map.Entry entry = iterator.next(); @@ -97,12 +97,9 @@ private void ttlCheck() throws InterruptedException { if (entry.getValue().closeIfOutOfTimeToLive()) { iterator.remove(); } else { - logger.ifPresent( - l -> - l.info( - "Pipe file (file name: {}) is still referenced {} times", - entry.getKey(), - entry.getValue().getReferenceCount())); + logBuilder.append( + String.format( + "<%s , %d times> ", entry.getKey(), entry.getValue().getReferenceCount())); } } catch (final IOException e) { LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", e); @@ -110,6 +107,9 @@ private void ttlCheck() throws InterruptedException { segmentLock.unlock(new File(hardlinkOrCopiedFile)); } } + if (logBuilder.length() > 0) { + logger.ifPresent(l -> l.info("Pipe file {}are still referenced", logBuilder)); + } } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java index 9bc3a4a030aa..9c51d79daad4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java @@ -73,6 +73,7 @@ private void ttlCheck() { PipeConfig.getInstance().getPipeWalPinMaxLogIntervalRounds(), memtableIdToPipeWALResourceMap.size()); + final StringBuilder logBuilder = new StringBuilder(); try { while (iterator.hasNext()) { final Map.Entry entry = iterator.next(); @@ -84,12 +85,9 @@ private void ttlCheck() { if (entry.getValue().invalidateIfPossible()) { iterator.remove(); } else { - logger.ifPresent( - l -> - l.info( - "WAL (memtableId {}) is still referenced {} times", - entry.getKey(), - entry.getValue().getReferenceCount())); + logBuilder.append( + String.format( + "<%d , %d times> ", entry.getKey(), entry.getValue().getReferenceCount())); } } finally { lock.unlock(); @@ -99,6 +97,10 @@ private void ttlCheck() { LOGGER.error( "Concurrent modification issues happened, skipping the WAL in this round of ttl check", e); + } finally { + if (logBuilder.length() > 0) { + logger.ifPresent(l -> l.info("WAL {}are still referenced", logBuilder)); + } } } From 3aef6ada2cf7dc6ec068ca44c82a87b90c8cb92d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:10:22 +0800 Subject: [PATCH 6/6] Pipe: Fixed the bug that historical aligned timeseries' attributes/tags/alias is not transferred (#14071) --- .../it/manual/IoTDBPipeMetaHistoricalIT.java | 56 +++++++++++++++++++ .../schemaregion/SchemaExecutionVisitor.java | 32 ++++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java index b6fd01a8efe7..2ce13afbf728 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java @@ -261,4 +261,60 @@ public void testAuthInclusion() throws Exception { new HashSet<>(Arrays.asList("admin,", "test,"))); } } + + @Test + public void testTimeSeriesInclusion() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create database root.sg", + "create timeseries root.sg.a.b int32", + "create aligned timeseries root.sg.`apache|timecho-tag-attr`.d1(s1 INT32 tags(tag1=v1, tag2=v2) attributes(attr1=v1, attr2=v2), s2 DOUBLE tags(tag3=v3, tag4=v4) attributes(attr3=v3, attr4=v4))"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "schema"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeseries", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + new HashSet<>( + Arrays.asList( + "root.sg.a.b,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.`apache|timecho-tag-attr`.d1.s1,null,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},{\"attr2\":\"v2\",\"attr1\":\"v1\"},null,null,BASE,", + "root.sg.`apache|timecho-tag-attr`.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,{\"tag4\":\"v4\",\"tag3\":\"v3\"},{\"attr4\":\"v4\",\"attr3\":\"v3\"},null,null,BASE,"))); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 22804b2379cd..a6caba0a3f24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -88,6 +88,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; /** Schema write {@link PlanNode} visitor */ public class SchemaExecutionVisitor extends PlanVisitor { @@ -286,6 +287,10 @@ private void executeInternalCreateAlignedTimeSeries( final List dataTypeList = measurementGroup.getDataTypes(); final List encodingList = measurementGroup.getEncodings(); final List compressionTypeList = measurementGroup.getCompressors(); + final List aliasList = measurementGroup.getAliasList(); + final List> tagsList = measurementGroup.getTagsList(); + final List> attributesList = measurementGroup.getAttributesList(); + final ICreateAlignedTimeSeriesPlan createAlignedTimeSeriesPlan = SchemaRegionWritePlanFactory.getCreateAlignedTimeSeriesPlan( devicePath, @@ -293,9 +298,10 @@ private void executeInternalCreateAlignedTimeSeries( dataTypeList, encodingList, compressionTypeList, - null, - null, - null); + aliasList, + tagsList, + attributesList); + // With merge is only true for pipe to upsert the receiver alias/tags/attributes in historical // transfer. // For normal internal creation, the alias/tags/attributes are not set @@ -322,6 +328,16 @@ private void executeInternalCreateAlignedTimeSeries( encodingList.remove(index); compressionTypeList.remove(index); + if (Objects.nonNull(aliasList)) { + aliasList.remove(index); + } + if (Objects.nonNull(tagsList)) { + tagsList.remove(index); + } + if (Objects.nonNull(attributesList)) { + attributesList.remove(index); + } + // If with merge is set, the lists are deep copied and need to be altered here. // We still remove the element from the original list to help cascading pipe transfer // schema. @@ -332,6 +348,16 @@ private void executeInternalCreateAlignedTimeSeries( createAlignedTimeSeriesPlan.getDataTypes().remove(index); createAlignedTimeSeriesPlan.getEncodings().remove(index); createAlignedTimeSeriesPlan.getCompressors().remove(index); + + if (Objects.nonNull(aliasList)) { + createAlignedTimeSeriesPlan.getAliasList().remove(index); + } + if (Objects.nonNull(tagsList)) { + createAlignedTimeSeriesPlan.getTagsList().remove(index); + } + if (Objects.nonNull(attributesList)) { + createAlignedTimeSeriesPlan.getAttributesList().remove(index); + } } if (measurementList.isEmpty()) {