Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pipe: Fix the problem that SessionUtils calculates Tablet without handling null pointers and update tsfile version #13990

Merged
merged 5 commits into from
Nov 7, 2024
Merged

Pipe: Fix the problem that SessionUtils calculates Tablet without handling null pointers and update tsfile version #13990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
698c5a3
update tsfile version
luoluoyuyu Nov 4, 2024
7e115eb
fix
luoluoyuyu Nov 5, 2024
7fc7691
fix
luoluoyuyu Nov 6, 2024
dba09d4
update SessionUtils
luoluoyuyu Nov 6, 2024
776e9d9
update tsfile version
luoluoyuyu Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 57 additions & 6 deletions .../java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ public abstract class TabletInsertionEventParser {

private static final Logger LOGGER = LoggerFactory.getLogger(TabletInsertionEventParser.class);

private static final LocalDate EMPTY_LOCALDATE = LocalDate.of(1000, 1, 1);

protected final PipeTaskMeta pipeTaskMeta; // used to report progress
protected final EnrichedEvent
sourceEvent; // used to report progress and filter value columns by time range
Expand Down Expand Up @@ -246,8 +248,12 @@ protected void parse(final InsertTabletNode insertTabletNode) {
final BitMap bitMap = new BitMap(this.timestampColumn.length);
if (Objects.isNull(originValueColumns[i])
|| Objects.isNull(originValueColumnDataTypes[i])) {
this.valueColumns[filteredColumnIndex] = null;
bitMap.markAll();
fillNullValue(
originValueColumnDataTypes[i],
this.valueColumns,
bitMap,
filteredColumnIndex,
rowIndexList.size());
} else {
this.valueColumns[filteredColumnIndex] =
filterValueColumnsByRowIndexList(
Expand Down Expand Up @@ -348,8 +354,12 @@ protected void parse(final Tablet tablet, final boolean isAligned) {
final BitMap bitMap = new BitMap(this.timestampColumn.length);
if (Objects.isNull(originValueColumns[i])
|| Objects.isNull(originValueColumnDataTypes[i])) {
this.valueColumns[filteredColumnIndex] = null;
bitMap.markAll();
fillNullValue(
originValueColumnDataTypes[i],
this.valueColumns,
bitMap,
filteredColumnIndex,
rowIndexList.size());
} else {
this.valueColumns[filteredColumnIndex] =
filterValueColumnsByRowIndexList(
Expand Down Expand Up @@ -449,7 +459,7 @@ private static Object filterValueColumnsByRowIndexList(

for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = LocalDate.MIN;
valueColumns[i] = EMPTY_LOCALDATE;
nullValueColumnBitmap.mark(i);
} else {
valueColumns[i] = dateValueColumns[rowIndexList.get(i)];
Expand All @@ -463,7 +473,7 @@ private static Object filterValueColumnsByRowIndexList(
: (int[]) originValueColumn;
for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = LocalDate.MIN;
valueColumns[i] = EMPTY_LOCALDATE;
nullValueColumnBitmap.mark(i);
} else {
valueColumns[i] =
Expand Down Expand Up @@ -569,6 +579,47 @@ private static Object filterValueColumnsByRowIndexList(
}
}

private void fillNullValue(
final TSDataType type,
final Object[] valueColumns,
final BitMap nullValueColumnBitmap,
final int columnIndex,
final int rowSize) {
nullValueColumnBitmap.markAll();
if (Objects.isNull(type)) {
return;
}
switch (type) {
case TIMESTAMP:
case INT64:
valueColumns[columnIndex] = new long[rowSize];
break;
case INT32:
valueColumns[columnIndex] = new int[rowSize];
break;
case DOUBLE:
valueColumns[columnIndex] = new double[rowSize];
break;
case FLOAT:
valueColumns[columnIndex] = new float[rowSize];
break;
case BOOLEAN:
valueColumns[columnIndex] = new boolean[rowSize];
break;
case DATE:
valueColumns[columnIndex] = new LocalDate[rowSize];
break;
case TEXT:
case BLOB:
case STRING:
valueColumns[columnIndex] = new Binary[rowSize];
break;
default:
throw new UnSupportedDataTypeException(
String.format("Data type %s is not supported.", type));
}
}

//////////////////////////// process ////////////////////////////

public abstract List<TabletInsertionEvent> processRowByRow(
Expand Down
2 changes: 1 addition & 1 deletion ...eengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void addTimeChunkToCache(String file, long offset, Chunk chunk) {
chunk.getData(),
chunk.getDeleteIntervalList(),
chunk.getChunkStatistic(),
chunk.getDecryptor()));
chunk.getEncryptParam()));
cachedTimeChunkSize += chunk.getHeader().getDataSize();
}

Expand Down
2 changes: 1 addition & 1 deletion ...ava/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ public void testOneUncompletedInMultiCompletedFiles2() throws IOException {
writeOneFile(path);
if (i == 5) {
RandomAccessFile randomAccessFile = new RandomAccessFile(path, "rw");
randomAccessFile.seek(randomAccessFile.length() - 100);
randomAccessFile.seek(randomAccessFile.length() - 130);
randomAccessFile.write(new byte[] {1, 2, 3, 4, 5, 6, 7, 8});
randomAccessFile.close();
}
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
<tsfile.version>1.2.0-1c9924b4-SNAPSHOT</tsfile.version>
<tsfile.version>1.2.0-241106-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
Expand Down
Loading