diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java index cfb511c3fccb..d00cecba128f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java @@ -17,6 +17,9 @@ import com.google.common.collect.Iterables; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestFile; import org.apache.iceberg.io.BulkDeletionFailureException; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; @@ -28,6 +31,7 @@ import java.util.Map; import java.util.stream.Stream; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -88,6 +92,36 @@ public void deleteFiles(Iterable pathsToDelete) partitions.forEach(this::deleteBatch); } + @Override + public InputFile newInputFile(ManifestFile manifest) + { + checkArgument( + manifest.keyMetadata() == null, + "Cannot decrypt manifest: %s (use EncryptingFileIO)", + manifest.path()); + return newInputFile(manifest.path(), manifest.length()); + } + + @Override + public InputFile newInputFile(DataFile file) + { + checkArgument( + file.keyMetadata() == null, + "Cannot decrypt data file: %s (use EncryptingFileIO)", + file.path()); + return newInputFile(file.path().toString(), file.fileSizeInBytes()); + } + + @Override + public InputFile newInputFile(DeleteFile file) + { + checkArgument( + file.keyMetadata() == null, + "Cannot decrypt delete file: %s (use EncryptingFileIO)", + file.path()); + return newInputFile(file.path().toString(), file.fileSizeInBytes()); + } + private void deleteBatch(List filesToDelete) { try { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergFileOperations.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergFileOperations.java index 36636217fe2b..1bcb313d4587 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergFileOperations.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergFileOperations.java @@ -183,7 +183,6 @@ public void testCreateOrReplaceTableAsSelect() .add(new FileOperation(SNAPSHOT, "OutputFile.create")) .add(new FileOperation(MANIFEST, "OutputFile.create")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .add(new FileOperation(STATS, "OutputFile.create")) .build()); } @@ -198,7 +197,6 @@ public void testSelect() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); } @@ -225,7 +223,6 @@ public void testSelectWithLimit(int numberOfFiles) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), min(icebergManifestPrefetching, numberOfFiles)) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), min(icebergManifestPrefetching, numberOfFiles)) .build()); assertFileSystemAccesses("EXPLAIN SELECT * FROM test_select_with_limit LIMIT 3", @@ -234,7 +231,6 @@ public void testSelectWithLimit(int numberOfFiles) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), numberOfFiles) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), numberOfFiles) .build()); assertFileSystemAccesses("EXPLAIN ANALYZE SELECT * FROM test_select_with_limit LIMIT 3", @@ -243,7 +239,6 @@ public void testSelectWithLimit(int numberOfFiles) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), numberOfFiles + min(icebergManifestPrefetching, numberOfFiles)) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), numberOfFiles + min(icebergManifestPrefetching, numberOfFiles)) .build()); assertUpdate("DROP TABLE test_select_with_limit"); @@ -276,7 +271,6 @@ public void testReadWholePartition() ALL_FILES, ImmutableMultiset.builder() .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -289,7 +283,6 @@ public void testReadWholePartition() ALL_FILES, ImmutableMultiset.builder() .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -301,7 +294,6 @@ public void testReadWholePartition() ALL_FILES, ImmutableMultiset.builder() .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -313,7 +305,6 @@ public void testReadWholePartition() ALL_FILES, ImmutableMultiset.builder() .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -328,7 +319,6 @@ public void testReadWholePartition() ALL_FILES, ImmutableMultiset.builder() .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -365,7 +355,6 @@ public void testReadWholePartitionSplittableFile() ALL_FILES, ImmutableMultiset.builder() .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -378,7 +367,6 @@ public void testReadWholePartitionSplittableFile() ALL_FILES, ImmutableMultiset.builder() .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .add(new FileOperation(METADATA_JSON, "InputFile.newStream")) .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) @@ -409,7 +397,6 @@ public void testSelectFromVersionedTable() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); assertFileSystemAccesses("SELECT * FROM " + tableName + " FOR VERSION AS OF " + v3SnapshotId, ImmutableMultiset.builder() @@ -417,7 +404,6 @@ public void testSelectFromVersionedTable() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .build()); assertFileSystemAccesses("SELECT * FROM " + tableName, ImmutableMultiset.builder() @@ -425,7 +411,6 @@ public void testSelectFromVersionedTable() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .build()); } @@ -452,7 +437,6 @@ public void testSelectFromVersionedTableWithSchemaEvolution() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); assertFileSystemAccesses("SELECT * FROM " + tableName + " FOR VERSION AS OF " + v3SnapshotId, ImmutableMultiset.builder() @@ -460,7 +444,6 @@ public void testSelectFromVersionedTableWithSchemaEvolution() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .build()); assertFileSystemAccesses("SELECT * FROM " + tableName, ImmutableMultiset.builder() @@ -468,7 +451,6 @@ public void testSelectFromVersionedTableWithSchemaEvolution() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .build()); } @@ -482,7 +464,6 @@ public void testSelectWithFilter() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); } @@ -498,7 +479,6 @@ public void testJoin() .addCopies(new FileOperation(SNAPSHOT, "InputFile.length"), 2) .addCopies(new FileOperation(SNAPSHOT, "InputFile.newStream"), 2) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 4) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 4) .build()); } @@ -516,7 +496,6 @@ public void testJoinWithPartitionedTable() .addCopies(new FileOperation(SNAPSHOT, "InputFile.length"), 2) .addCopies(new FileOperation(SNAPSHOT, "InputFile.newStream"), 2) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 4) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 4) .build()); } @@ -531,7 +510,6 @@ public void testExplainSelect() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); } @@ -546,7 +524,6 @@ public void testShowStatsForTable() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); } @@ -563,7 +540,6 @@ public void testShowStatsForPartitionedTable() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); } @@ -578,7 +554,6 @@ public void testShowStatsForTableWithFilter() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); } @@ -595,7 +570,6 @@ public void testPredicateWithVarcharCastToDate() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 2) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 2) .build()); // CAST to date and comparison @@ -605,7 +579,6 @@ public void testPredicateWithVarcharCastToDate() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) // fewer than without filter - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); // CAST to date and BETWEEN @@ -615,7 +588,6 @@ public void testPredicateWithVarcharCastToDate() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) // fewer than without filter - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); // conversion to date as a date function @@ -625,7 +597,6 @@ public void testPredicateWithVarcharCastToDate() .add(new FileOperation(SNAPSHOT, "InputFile.length")) .add(new FileOperation(SNAPSHOT, "InputFile.newStream")) .add(new FileOperation(MANIFEST, "InputFile.newStream")) // fewer than without filter - .add(new FileOperation(MANIFEST, "InputFile.length")) .build()); assertUpdate("DROP TABLE test_varchar_as_date_predicate"); @@ -651,7 +622,6 @@ public void testRemoveOrphanFiles() .addCopies(new FileOperation(SNAPSHOT, "InputFile.length"), 4) .addCopies(new FileOperation(SNAPSHOT, "InputFile.newStream"), 4) .addCopies(new FileOperation(MANIFEST, "InputFile.newStream"), 5) - .addCopies(new FileOperation(MANIFEST, "InputFile.length"), 5) .build()); assertUpdate("DROP TABLE " + tableName);