Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ignore nested delta directories for Hive ACID #20840

Merged
merged 1 commit into from
Mar 6, 2024
Merged

Ignore nested delta directories for Hive ACID #20840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import io.trino.filesystem.TrinoFileSystem;
import io.trino.filesystem.TrinoInputFile;
import io.trino.filesystem.TrinoOutputFile;
import io.trino.spi.TrinoException;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -38,7 +37,6 @@
import static com.google.common.collect.Multimaps.asMap;
import static io.trino.hive.thrift.metastore.hive_metastoreConstants.TABLE_IS_TRANSACTIONAL;
import static io.trino.hive.thrift.metastore.hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES;
import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES;
import static java.lang.Integer.parseInt;
import static java.lang.Long.parseLong;
import static java.nio.charset.StandardCharsets.UTF_8;
Expand Down Expand Up @@ -133,7 +131,8 @@ public static AcidState getAcidState(TrinoFileSystem fileSystem, Location direct

if (name.startsWith("base_") || name.startsWith("delta_") || name.startsWith("delete_delta_")) {
if (suffix.indexOf('/', slash + 1) != -1) {
throw new TrinoException(HIVE_INVALID_BUCKET_FILES, "Found file in sub-directory of ACID directory: " + file.location());
// Ignore nested delta directories
electrum marked this conversation as resolved.
Show resolved Hide resolved
continue;
}
groupedFiles.put(name, file);
}
Expand Down
28 changes: 28 additions & 0 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,34 @@ public void testDeleteDeltaSubdirPathGeneration()
assertThat(deleteDeltaSubdirPath).isEqualTo("delete_delta_0000013_0000013_0005");
}

@Test
public void testSkippingSubDirectories()
pajaks marked this conversation as resolved.
Show resolved Hide resolved
throws IOException
{
TrinoFileSystem fileSystem = new MemoryFileSystem();
createFile(fileSystem, "memory:///tbl/part1/base_1/bucket_0", FAKE_DATA);
createFile(fileSystem, "memory:///tbl/part1/base_1/base_1/bucket_0", FAKE_DATA);
createFile(fileSystem, "memory:///tbl/part1/delta_025_025/bucket_0", FAKE_DATA);
createFile(fileSystem, "memory:///tbl/part1/delta_025_025/delta_025_025/bucket_0", FAKE_DATA);
createFile(fileSystem, "memory:///tbl/part1/delete_delta_029_029/bucket_0", FAKE_DATA);
createFile(fileSystem, "memory:///tbl/part1/delete_delta_029_029/delete_delta_029_029/bucket_0", FAKE_DATA);

AcidState state = getAcidState(
fileSystem,
Location.of("memory:///tbl/part1"),
new ValidWriteIdList("tbl:100:%d:".formatted(Long.MAX_VALUE)));

// Subdirectories in base directory should be skipped similar to Hive implementation
assertThat(state.baseDirectory()).contains(Location.of("memory:///tbl/part1/base_1"));
assertThat(state.originalFiles()).isEmpty();

// Subdirectories in delta directories should be skipped similar to Hive implementation
List<ParsedDelta> deltas = state.deltas();
assertThat(deltas.size()).isEqualTo(2);
assertThat(deltas.get(0).path()).isEqualTo("memory:///tbl/part1/delta_025_025");
assertThat(deltas.get(1).path()).isEqualTo("memory:///tbl/part1/delete_delta_029_029");
}

private static void createFile(TrinoFileSystem fileSystem, String location, byte[] data)
throws IOException
{
Expand Down
25 changes: 25 additions & 0 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.junit.jupiter.api.Test;

import java.io.IOException;
import java.util.List;

import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -453,4 +454,28 @@ public void testDeleteDeltaSubdirPathGeneration()
deleteDeltaSubdirPath = AcidUtils.deleteDeltaSubdir(1, 10, 5);
assertThat(deleteDeltaSubdirPath).isEqualTo("delete_delta_0000001_0000010_0005");
}

@Test
public void testSkippingSubDirectories()
throws IOException
{
Configuration conf = new Configuration(false);
MockFileSystem fs = new MockFileSystem(conf,
new MockFile("mock:/tbl/part1/base_1/bucket_0", 500, new byte[0]),
new MockFile("mock:/tbl/part1/base_1/base_1/bucket_0", 500, new byte[0]),
new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 500, new byte[0]),
new MockFile("mock:/tbl/part1/delta_025_025/delta_025_025/bucket_0", 500, new byte[0]),
new MockFile("mock:/tbl/part1/delete_delta_029_029/bucket_0", 500, new byte[0]),
new MockFile("mock:/tbl/part1/delete_delta_029_029/delete_delta_029_029/bucket_0", 500, new byte[0]));

Path part = new MockPath(fs, "mock:/tbl/part1");
AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":"));
assertThat(dir.getBaseDirectory().toString()).isEqualTo("mock:/tbl/part1/base_1");
List<FileStatus> obsolete = dir.getObsolete();
assertThat(obsolete.size()).isEqualTo(0);
List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
assertThat(delts.size()).isEqualTo(2);
assertThat(delts.get(0).getPath().toString()).isEqualTo("mock:/tbl/part1/delta_025_025");
assertThat(delts.get(1).getPath().toString()).isEqualTo("mock:/tbl/part1/delete_delta_029_029");
}
}
Loading