From e3923e9e69a4d2591513642ab47341a9676b0d57 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 29 Mar 2023 00:49:11 -0700 Subject: [PATCH] Fix handling of Hive ACID tables with hidden directories --- .../src/main/java/io/trino/plugin/hive/util/AcidTables.java | 3 +-- .../test/java/io/trino/plugin/hive/util/TestAcidTables.java | 6 ++++++ .../java/io/trino/plugin/hive/util/TestHiveAcidUtils.java | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java index 830b79c2c36f..ebb5b7a80047 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java @@ -312,8 +312,7 @@ private static List listFiles(TrinoFileSystem fileSystem, String dire FileIterator iterator = fileSystem.listFiles(directory); while (iterator.hasNext()) { FileEntry file = iterator.next(); - String name = new Path(file.location()).getName(); - if (!name.startsWith("_") && !name.startsWith(".")) { + if (!file.location().contains("/_") && !file.location().contains("/.")) { files.add(file); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java index 369ee0745feb..7efee32be0c7 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java @@ -117,6 +117,8 @@ public void testOriginal() new MockFile("mock:/tbl/part1/000002_0", 500, FAKE_DATA), new MockFile("mock:/tbl/part1/random", 500, FAKE_DATA), new MockFile("mock:/tbl/part1/_done", 0, FAKE_DATA), + new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, FAKE_DATA), + new MockFile("mock:/tbl/part1/_tmp/abc/000000_0", 0, FAKE_DATA), new MockFile("mock:/tbl/part1/subdir/000000_0", 0, FAKE_DATA)); AcidState state = getAcidState( testingTrinoFileSystem(fs), @@ -147,6 +149,8 @@ public void testOriginalDeltas() new MockFile("mock:/tbl/part1/000002_0", 500, FAKE_DATA), new MockFile("mock:/tbl/part1/random", 500, FAKE_DATA), new MockFile("mock:/tbl/part1/_done", 0, FAKE_DATA), + new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, FAKE_DATA), + new MockFile("mock:/tbl/part1/_tmp/delta_025_025/000000_0", 0, FAKE_DATA), new MockFile("mock:/tbl/part1/subdir/000000_0", 0, FAKE_DATA), new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 0, FAKE_DATA), new MockFile("mock:/tbl/part1/delta_029_029/bucket_0", 0, FAKE_DATA), @@ -185,6 +189,8 @@ public void testBaseDeltas() throws Exception { MockFileSystem fs = new MockFileSystem(newEmptyConfiguration(), + new MockFile("mock:/tbl/part1/_tmp/bucket_0", 0, FAKE_DATA), + new MockFile("mock:/tbl/part1/_tmp/base_5/bucket_0", 0, FAKE_DATA), new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, FAKE_DATA), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, FAKE_DATA), new MockFile("mock:/tbl/part1/base_49/bucket_0", 500, FAKE_DATA), diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java index b978e7e472a6..1e7ae4b749b7 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java @@ -53,6 +53,8 @@ public void testOriginal() new MockFile("mock:/tbl/part1/000002_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/random", 500, new byte[0]), new MockFile("mock:/tbl/part1/_done", 0, new byte[0]), + new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, new byte[0]), + new MockFile("mock:/tbl/part1/_tmp/abc/000000_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/subdir/000000_0", 0, new byte[0])); AcidUtils.Directory dir = AcidUtils.getAcidState( new MockPath(fs, "/tbl/part1"), @@ -83,6 +85,8 @@ public void testOriginalDeltas() new MockFile("mock:/tbl/part1/000002_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/random", 500, new byte[0]), new MockFile("mock:/tbl/part1/_done", 0, new byte[0]), + new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, new byte[0]), + new MockFile("mock:/tbl/part1/_tmp/delta_025_025/000000_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/subdir/000000_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_029_029/bucket_0", 0, new byte[0]), @@ -123,6 +127,8 @@ public void testBaseDeltas() { Configuration conf = newEmptyConfiguration(); MockFileSystem fs = new MockFileSystem(conf, + new MockFile("mock:/tbl/part1/_tmp/bucket_0", 0, new byte[0]), + new MockFile("mock:/tbl/part1/_tmp/base_5/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_49/bucket_0", 500, new byte[0]),