Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Iceberg predicate pushdown #1561

Merged
merged 1 commit into from
Oct 22, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -38,12 +38,15 @@
import io.prestosql.spi.connector.ConnectorTableHandle;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorTableProperties;
import io.prestosql.spi.connector.Constraint;
import io.prestosql.spi.connector.ConstraintApplicationResult;
import io.prestosql.spi.connector.SchemaNotFoundException;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.connector.SchemaTablePrefix;
import io.prestosql.spi.connector.SystemTable;
import io.prestosql.spi.connector.TableNotFoundException;
import io.prestosql.spi.connector.classloader.ClassLoaderSafeSystemTable;
import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.spi.statistics.ComputedStatistics;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.TimestampType;
@@ -78,6 +81,7 @@
import static io.prestosql.plugin.hive.HiveColumnHandle.updateRowIdHandle;
import static io.prestosql.plugin.hive.HiveSchemaProperties.getLocation;
import static io.prestosql.plugin.hive.util.HiveWriteUtils.getTableDefaultLocation;
import static io.prestosql.plugin.iceberg.DomainConverter.convertTupleDomainTypes;
import static io.prestosql.plugin.iceberg.ExpressionConverter.toIcebergExpression;
import static io.prestosql.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY;
import static io.prestosql.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY;
@@ -521,4 +525,23 @@ public void rollback()
{
// TODO: cleanup open transaction
}

@Override
public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint)
{
IcebergTableHandle table = (IcebergTableHandle) handle;
TupleDomain<HiveColumnHandle> newDomain = convertTupleDomainTypes(constraint.getSummary().transform(HiveColumnHandle.class::cast));

if (newDomain.equals(table.getPredicate())) {
return Optional.empty();
}

return Optional.of(new ConstraintApplicationResult<>(
new IcebergTableHandle(table.getSchemaName(),
table.getTableName(),
table.getTableType(),
table.getSnapshotId(),
table.getPredicate().intersect(newDomain)),
constraint.getSummary()));
}
}
Original file line number Diff line number Diff line change
@@ -47,7 +47,6 @@
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.Iterators.limit;
import static io.prestosql.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION;
import static io.prestosql.plugin.iceberg.DomainConverter.convertTupleDomainTypes;
import static io.prestosql.plugin.iceberg.IcebergUtil.getIdentityPartitions;
import static java.lang.Math.toIntExact;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -90,7 +89,6 @@ public CompletableFuture<ConnectorSplitBatch> getNextBatch(ConnectorPartitionHan
{
// TODO: move this to a background thread
List<ConnectorSplit> splits = new ArrayList<>();
TupleDomain<HiveColumnHandle> predicate = convertTupleDomainTypes(this.predicate);
Iterator<FileScanTask> iterator = limit(fileScanIterator, maxSize);
while (iterator.hasNext()) {
FileScanTask task = iterator.next();