Fix handling of Hive ACID tables with hidden directories

trinodb · Apr 4, 2023 · 76a98c7 · 76a98c7
1 parent 309434f
commit 76a98c7
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/AcidTables.java
@@ -312,8 +312,7 @@ private static List<FileEntry> listFiles(TrinoFileSystem fileSystem, String dire
         FileIterator iterator = fileSystem.listFiles(directory);
         while (iterator.hasNext()) {
             FileEntry file = iterator.next();
-            String name = new Path(file.location()).getName();
-            if (!name.startsWith("_") && !name.startsWith(".")) {
+            if (!file.location().contains("/_") && !file.location().contains("/.")) {
                 files.add(file);
             }
         }

diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestAcidTables.java
@@ -117,6 +117,8 @@ public void testOriginal()
                 new MockFile("mock:/tbl/part1/000002_0", 500, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/random", 500, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/_done", 0, FAKE_DATA),
+                new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, FAKE_DATA),
+                new MockFile("mock:/tbl/part1/_tmp/abc/000000_0", 0, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/subdir/000000_0", 0, FAKE_DATA));
         AcidState state = getAcidState(
                 testingTrinoFileSystem(fs),
@@ -147,6 +149,8 @@ public void testOriginalDeltas()
                 new MockFile("mock:/tbl/part1/000002_0", 500, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/random", 500, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/_done", 0, FAKE_DATA),
+                new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, FAKE_DATA),
+                new MockFile("mock:/tbl/part1/_tmp/delta_025_025/000000_0", 0, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/subdir/000000_0", 0, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 0, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/delta_029_029/bucket_0", 0, FAKE_DATA),
@@ -185,6 +189,8 @@ public void testBaseDeltas()
             throws Exception
     {
         MockFileSystem fs = new MockFileSystem(newEmptyConfiguration(),
+                new MockFile("mock:/tbl/part1/_tmp/bucket_0", 0, FAKE_DATA),
+                new MockFile("mock:/tbl/part1/_tmp/base_5/bucket_0", 0, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, FAKE_DATA),
                 new MockFile("mock:/tbl/part1/base_49/bucket_0", 500, FAKE_DATA),

diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveAcidUtils.java
@@ -53,6 +53,8 @@ public void testOriginal()
                 new MockFile("mock:/tbl/part1/000002_0", 500, new byte[0]),
                 new MockFile("mock:/tbl/part1/random", 500, new byte[0]),
                 new MockFile("mock:/tbl/part1/_done", 0, new byte[0]),
+                new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, new byte[0]),
+                new MockFile("mock:/tbl/part1/_tmp/abc/000000_0", 0, new byte[0]),
                 new MockFile("mock:/tbl/part1/subdir/000000_0", 0, new byte[0]));
         AcidUtils.Directory dir = AcidUtils.getAcidState(
                 new MockPath(fs, "/tbl/part1"),
@@ -83,6 +85,8 @@ public void testOriginalDeltas()
                 new MockFile("mock:/tbl/part1/000002_0", 500, new byte[0]),
                 new MockFile("mock:/tbl/part1/random", 500, new byte[0]),
                 new MockFile("mock:/tbl/part1/_done", 0, new byte[0]),
+                new MockFile("mock:/tbl/part1/_tmp/000000_0", 0, new byte[0]),
+                new MockFile("mock:/tbl/part1/_tmp/delta_025_025/000000_0", 0, new byte[0]),
                 new MockFile("mock:/tbl/part1/subdir/000000_0", 0, new byte[0]),
                 new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 0, new byte[0]),
                 new MockFile("mock:/tbl/part1/delta_029_029/bucket_0", 0, new byte[0]),
@@ -123,6 +127,8 @@ public void testBaseDeltas()
     {
         Configuration conf = newEmptyConfiguration();
         MockFileSystem fs = new MockFileSystem(conf,
+                new MockFile("mock:/tbl/part1/_tmp/bucket_0", 0, new byte[0]),
+                new MockFile("mock:/tbl/part1/_tmp/base_5/bucket_0", 0, new byte[0]),
                 new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]),
                 new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]),
                 new MockFile("mock:/tbl/part1/base_49/bucket_0", 500, new byte[0]),