diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 0af56e0ac0b5..e76862afed1c 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -267,6 +267,7 @@ private synchronized void addConnectorInternal(MaterializedConnector connector) metadataManager.getTablePropertyManager().addProperties(connectorId, connector.getTableProperties()); metadataManager.getColumnPropertyManager().addProperties(connectorId, connector.getColumnProperties()); metadataManager.getSchemaPropertyManager().addProperties(connectorId, connector.getSchemaProperties()); + metadataManager.getAnalyzePropertyManager().addProperties(connectorId, connector.getAnalyzeProperties()); metadataManager.getSessionPropertyManager().addConnectorSessionProperties(connectorId, connector.getSessionProperties()); } @@ -337,6 +338,7 @@ private static class MaterializedConnector private final List> tableProperties; private final List> schemaProperties; private final List> columnProperties; + private final List> analyzeProperties; public MaterializedConnector(ConnectorId connectorId, Connector connector) { @@ -425,6 +427,10 @@ public MaterializedConnector(ConnectorId connectorId, Connector connector) List> columnProperties = connector.getColumnProperties(); requireNonNull(columnProperties, "Connector %s returned a null column properties set"); this.columnProperties = ImmutableList.copyOf(columnProperties); + + List> analyzeProperties = connector.getAnalyzeProperties(); + requireNonNull(analyzeProperties, "Connector %s returned a null analyze properties set"); + this.analyzeProperties = ImmutableList.copyOf(analyzeProperties); } public ConnectorId getConnectorId() @@ -496,5 +502,10 @@ public List> getSchemaProperties() { return schemaProperties; } + + public List> getAnalyzeProperties() + { + return analyzeProperties; + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzeMetadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzeMetadata.java new file mode 100644 index 000000000000..4a9749a201e5 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzeMetadata.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.statistics.TableStatisticsMetadata; + +import static java.util.Objects.requireNonNull; + +public class AnalyzeMetadata +{ + private final TableStatisticsMetadata statisticsMetadata; + private final TableHandle tableHandle; + + public AnalyzeMetadata(TableStatisticsMetadata statisticsMetadata, TableHandle tableHandle) + { + this.statisticsMetadata = requireNonNull(statisticsMetadata, "statisticsMetadata is null"); + this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); + } + + public TableStatisticsMetadata getStatisticsMetadata() + { + return statisticsMetadata; + } + + public TableHandle getTableHandle() + { + return tableHandle; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzePropertyManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzePropertyManager.java new file mode 100644 index 000000000000..77cf02f50202 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzePropertyManager.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY; + +public class AnalyzePropertyManager + extends AbstractPropertyManager +{ + public AnalyzePropertyManager() + { + super("analyze", INVALID_ANALYZE_PROPERTY); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzeTableHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzeTableHandle.java new file mode 100644 index 000000000000..8e7cd5d278e8 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/AnalyzeTableHandle.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.connector.ConnectorId; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class AnalyzeTableHandle +{ + private final ConnectorId connectorId; + private final ConnectorTransactionHandle transactionHandle; + private final ConnectorTableHandle connectorHandle; + + @JsonCreator + public AnalyzeTableHandle( + @JsonProperty("connectorId") ConnectorId connectorId, + @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, + @JsonProperty("connectorHandle") ConnectorTableHandle connectorHandle) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); + this.connectorHandle = requireNonNull(connectorHandle, "connectorHandle is null"); + } + + @JsonProperty + public ConnectorId getConnectorId() + { + return connectorId; + } + + @JsonProperty + public ConnectorTableHandle getConnectorHandle() + { + return connectorHandle; + } + + @JsonProperty + public ConnectorTransactionHandle getTransactionHandle() + { + return transactionHandle; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AnalyzeTableHandle that = (AnalyzeTableHandle) o; + return Objects.equals(connectorId, that.connectorId) && + Objects.equals(transactionHandle, that.transactionHandle) && + Objects.equals(connectorHandle, that.connectorHandle); + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId, transactionHandle, connectorHandle); + } + + @Override + public String toString() + { + return connectorId + ":" + connectorHandle + ":" + transactionHandle; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java index 20720546d264..bd1f86603608 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -69,6 +69,8 @@ public interface Metadata Optional getSystemTable(Session session, QualifiedObjectName tableName); + Optional getTableHandleForStatisticsCollection(Session session, QualifiedObjectName tableName, Map analyzeProperties); + List getLayouts(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns); TableLayout getLayout(Session session, TableLayoutHandle handle); @@ -191,8 +193,23 @@ public interface Metadata /** * Describes statistics that must be collected during a write. */ + TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, String catalogName, ConnectorTableMetadata tableMetadata); + + /** + * Describe statistics that must be collected during a statistics collection + */ TableStatisticsMetadata getStatisticsCollectionMetadata(Session session, String catalogName, ConnectorTableMetadata tableMetadata); + /** + * Begin statistics collection + */ + AnalyzeTableHandle beginStatisticsCollection(Session session, TableHandle tableHandle); + + /** + * Finish statistics collection + */ + void finishStatisticsCollection(Session session, AnalyzeTableHandle tableHandle, Collection computedStatistics); + /** * Start a SELECT/UPDATE/INSERT/DELETE query */ @@ -313,4 +330,6 @@ public interface Metadata TablePropertyManager getTablePropertyManager(); ColumnPropertyManager getColumnPropertyManager(); + + AnalyzePropertyManager getAnalyzePropertyManager(); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java index ec54c2015d72..eaa7eb7e44d8 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -120,6 +120,7 @@ public class MetadataManager private final SchemaPropertyManager schemaPropertyManager; private final TablePropertyManager tablePropertyManager; private final ColumnPropertyManager columnPropertyManager; + private final AnalyzePropertyManager analyzePropertyManager; private final TransactionManager transactionManager; private final ConcurrentMap> catalogsByQueryId = new ConcurrentHashMap<>(); @@ -131,6 +132,7 @@ public MetadataManager(FeaturesConfig featuresConfig, SchemaPropertyManager schemaPropertyManager, TablePropertyManager tablePropertyManager, ColumnPropertyManager columnPropertyManager, + AnalyzePropertyManager analyzePropertyManager, TransactionManager transactionManager) { this(featuresConfig, @@ -141,6 +143,7 @@ public MetadataManager(FeaturesConfig featuresConfig, schemaPropertyManager, tablePropertyManager, columnPropertyManager, + analyzePropertyManager, transactionManager); } @@ -153,6 +156,7 @@ public MetadataManager(FeaturesConfig featuresConfig, SchemaPropertyManager schemaPropertyManager, TablePropertyManager tablePropertyManager, ColumnPropertyManager columnPropertyManager, + AnalyzePropertyManager analyzePropertyManager, TransactionManager transactionManager) { functions = new FunctionRegistry(typeManager, blockEncodingSerde, featuresConfig); @@ -164,6 +168,7 @@ public MetadataManager(FeaturesConfig featuresConfig, this.schemaPropertyManager = requireNonNull(schemaPropertyManager, "schemaPropertyManager is null"); this.tablePropertyManager = requireNonNull(tablePropertyManager, "tablePropertyManager is null"); this.columnPropertyManager = requireNonNull(columnPropertyManager, "columnPropertyManager is null"); + this.analyzePropertyManager = requireNonNull(analyzePropertyManager, "analyzePropertyManager is null"); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); verifyComparableOrderableContract(); @@ -200,6 +205,7 @@ public static MetadataManager createTestMetadataManager(TransactionManager trans new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); } @@ -325,6 +331,25 @@ public Optional getTableHandle(Session session, QualifiedObjectName return Optional.empty(); } + @Override + public Optional getTableHandleForStatisticsCollection(Session session, QualifiedObjectName table, Map analyzeProperties) + { + requireNonNull(table, "table is null"); + + Optional catalog = getOptionalCatalogMetadata(session, table.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorId connectorId = catalogMetadata.getConnectorId(session, table); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + + ConnectorTableHandle tableHandle = metadata.getTableHandleForStatisticsCollection(session.toConnectorSession(connectorId), table.asSchemaTableName(), analyzeProperties); + if (tableHandle != null) { + return Optional.of(new TableHandle(connectorId, tableHandle)); + } + } + return Optional.empty(); + } + @Override public Optional getSystemTable(Session session, QualifiedObjectName tableName) { @@ -618,6 +643,15 @@ public Optional getInsertLayout(Session session, TableHandle tab .map(layout -> new NewTableLayout(connectorId, catalogMetadata.getTransactionHandleFor(connectorId), layout)); } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, String catalogName, ConnectorTableMetadata tableMetadata) + { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + return metadata.getStatisticsCollectionMetadataForWrite(session.toConnectorSession(connectorId), tableMetadata); + } + @Override public TableStatisticsMetadata getStatisticsCollectionMetadata(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { @@ -627,6 +661,26 @@ public TableStatisticsMetadata getStatisticsCollectionMetadata(Session session, return metadata.getStatisticsCollectionMetadata(session.toConnectorSession(connectorId), tableMetadata); } + @Override + public AnalyzeTableHandle beginStatisticsCollection(Session session, TableHandle tableHandle) + { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + + ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorTableHandle connectorTableHandle = metadata.beginStatisticsCollection(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + return new AnalyzeTableHandle(connectorId, transactionHandle, connectorTableHandle); + } + + @Override + public void finishStatisticsCollection(Session session, AnalyzeTableHandle tableHandle, Collection computedStatistics) + { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(), tableHandle.getConnectorHandle(), computedStatistics); + } + @Override public Optional getNewTableLayout(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { @@ -967,6 +1021,11 @@ public ColumnPropertyManager getColumnPropertyManager() return columnPropertyManager; } + public AnalyzePropertyManager getAnalyzePropertyManager() + { + return analyzePropertyManager; + } + private ViewDefinition deserializeView(String data) { try { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java new file mode 100644 index 000000000000..973f96d217c7 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java @@ -0,0 +1,197 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.statistics.ComputedStatistics; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.planner.plan.PlanNodeId; +import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class StatisticsWriterOperator + implements Operator +{ + public static final List TYPES = ImmutableList.of(BIGINT); + + public static class StatisticsWriterOperatorFactory + implements OperatorFactory + { + private final int operatorId; + private final PlanNodeId planNodeId; + private final StatisticsWriter statisticsWriter; + private final boolean rowCountEnabled; + private final StatisticAggregationsDescriptor descriptor; + private boolean closed; + + public StatisticsWriterOperatorFactory(int operatorId, PlanNodeId planNodeId, StatisticsWriter statisticsWriter, boolean rowCountEnabled, StatisticAggregationsDescriptor descriptor) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.statisticsWriter = requireNonNull(statisticsWriter, "statisticsWriter is null"); + this.rowCountEnabled = rowCountEnabled; + this.descriptor = requireNonNull(descriptor, "descriptor is null"); + } + + @Override + public Operator createOperator(DriverContext driverContext) + { + checkState(!closed, "Factory is already closed"); + OperatorContext context = driverContext.addOperatorContext(operatorId, planNodeId, StatisticsWriterOperator.class.getSimpleName()); + return new StatisticsWriterOperator(context, statisticsWriter, descriptor, rowCountEnabled); + } + + @Override + public void noMoreOperators() + { + closed = true; + } + + @Override + public OperatorFactory duplicate() + { + return new StatisticsWriterOperatorFactory(operatorId, planNodeId, statisticsWriter, rowCountEnabled, descriptor); + } + } + + private enum State + { + RUNNING, FINISHING, FINISHED + } + + private final OperatorContext operatorContext; + private final StatisticsWriter statisticsWriter; + private final StatisticAggregationsDescriptor descriptor; + private final boolean rowCountEnabled; + + private State state = State.RUNNING; + private final ImmutableList.Builder computedStatisticsBuilder = ImmutableList.builder(); + + public StatisticsWriterOperator(OperatorContext operatorContext, StatisticsWriter statisticsWriter, StatisticAggregationsDescriptor descriptor, boolean rowCountEnabled) + { + this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); + this.statisticsWriter = requireNonNull(statisticsWriter, "statisticsWriter is null"); + this.descriptor = requireNonNull(descriptor, "descriptor is null"); + this.rowCountEnabled = rowCountEnabled; + } + + @Override + public OperatorContext getOperatorContext() + { + return operatorContext; + } + + @Override + public boolean needsInput() + { + return state == State.RUNNING; + } + + @Override + public void addInput(Page page) + { + requireNonNull(page, "page is null"); + checkState(state == State.RUNNING, "Operator is %s", state); + + for (int position = 0; position < page.getPositionCount(); position++) { + computedStatisticsBuilder.add(getComputedStatistics(page, position)); + } + } + + @Override + public Page getOutput() + { + if (state != State.FINISHING) { + return null; + } + state = State.FINISHED; + + Collection computedStatistics = computedStatisticsBuilder.build(); + statisticsWriter.writeStatistics(computedStatistics); + + // output page will only be constructed once, + // so a new PageBuilder is constructed (instead of using PageBuilder.reset) + PageBuilder page = new PageBuilder(1, TYPES); + page.declarePosition(); + BlockBuilder rowsBuilder = page.getBlockBuilder(0); + if (rowCountEnabled) { + BIGINT.writeLong(rowsBuilder, getRowCount(computedStatistics)); + } + else { + rowsBuilder.appendNull(); + } + + return page.build(); + } + + @Override + public void finish() + { + if (state == State.RUNNING) { + state = State.FINISHING; + } + } + + @Override + public boolean isFinished() + { + return state == State.FINISHED; + } + + private ComputedStatistics getComputedStatistics(Page page, int position) + { + ImmutableList.Builder groupingColumns = ImmutableList.builder(); + ImmutableList.Builder groupingValues = ImmutableList.builder(); + descriptor.getGrouping().forEach((column, channel) -> { + groupingColumns.add(column); + groupingValues.add(page.getBlock(channel).getSingleValueBlock(position)); + }); + + ComputedStatistics.Builder statistics = ComputedStatistics.builder(groupingColumns.build(), groupingValues.build()); + + descriptor.getTableStatistics().forEach((type, channel) -> + statistics.addTableStatistic(type, page.getBlock(channel).getSingleValueBlock(position))); + + descriptor.getColumnStatistics().forEach((metadata, channel) -> statistics.addColumnStatistic(metadata, page.getBlock(channel).getSingleValueBlock(position))); + + return statistics.build(); + } + + private static long getRowCount(Collection computedStatistics) + { + return computedStatistics.stream() + .map(statistics -> statistics.getTableStatistics().get(ROW_COUNT)) + .filter(Objects::nonNull) + .mapToLong(block -> BIGINT.getLong(block, 0)) + .reduce((first, second) -> first + second) + .orElse(0L); + } + + public interface StatisticsWriter + { + void writeStatistics(Collection computedStatistics); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index d7a897f55776..3425159867a2 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -53,6 +53,7 @@ import com.facebook.presto.memory.MemoryResource; import com.facebook.presto.memory.NodeMemoryConfig; import com.facebook.presto.memory.ReservedSystemMemoryConfig; +import com.facebook.presto.metadata.AnalyzePropertyManager; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; import com.facebook.presto.metadata.DiscoveryNodeManager; @@ -220,6 +221,9 @@ protected void setup(Binder binder) // column properties binder.bind(ColumnPropertyManager.class).in(Scopes.SINGLETON); + // analyze properties + binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON); + // node manager discoveryBinder(binder).bindSelector("presto"); binder.bind(DiscoveryNodeManager.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java index 20c68cccd668..5ca74f928b64 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java @@ -130,6 +130,7 @@ public class Analysis private Optional createTableComment = Optional.empty(); private Optional insert = Optional.empty(); + private Optional analyzeTarget = Optional.empty(); // for describe input and describe output private final boolean isDescribe; @@ -522,6 +523,16 @@ public Optional getCreateTableDestination() return createTableDestination; } + public Optional getAnalyzeTarget() + { + return analyzeTarget; + } + + public void setAnalyzeTarget(TableHandle analyzeTarget) + { + this.analyzeTarget = Optional.of(analyzeTarget); + } + public void setCreateTableProperties(Map createTableProperties) { this.createTableProperties = ImmutableMap.copyOf(createTableProperties); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index 6de62fad2d21..9000fe7eb007 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionKind; import com.facebook.presto.metadata.Metadata; @@ -31,6 +32,7 @@ import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.security.AccessDeniedException; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.MapType; @@ -45,6 +47,7 @@ import com.facebook.presto.sql.tree.AddColumn; import com.facebook.presto.sql.tree.AliasedRelation; import com.facebook.presto.sql.tree.AllColumns; +import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateSchema; @@ -146,6 +149,7 @@ import static com.facebook.presto.metadata.FunctionKind.WINDOW; import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; @@ -209,6 +213,7 @@ import static com.google.common.collect.Iterables.getLast; import static com.google.common.collect.Iterables.transform; import static java.lang.Math.toIntExact; +import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; @@ -402,6 +407,49 @@ protected Scope visitDelete(Delete node, Optional scope) return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT)); } + @Override + protected Scope visitAnalyze(Analyze node, Optional scope) + { + analysis.setUpdateType("ANALYZE"); + QualifiedObjectName tableName = createQualifiedObjectName(session, node, node.getTableName()); + + // verify the target table exists and it's not a view + if (metadata.getView(session, tableName).isPresent()) { + throw new SemanticException(NOT_SUPPORTED, node, "Analyzing views is not supported"); + } + + validateProperties(node.getProperties(), scope); + ConnectorId connectorId = metadata.getCatalogHandle(session, tableName.getCatalogName()) + .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + tableName.getCatalogName())); + + Map analyzeProperties = metadata.getAnalyzePropertyManager().getProperties( + connectorId, + connectorId.getCatalogName(), + mapFromProperties(node.getProperties()), + session, + metadata, + analysis.getParameters()); + TableHandle tableHandle = metadata.getTableHandleForStatisticsCollection(session, tableName, analyzeProperties) + .orElseThrow(() -> (new SemanticException(MISSING_TABLE, node, "Table '%s' does not exist", tableName))); + + // user must have read and insert permission in order to analyze stats of a table + analysis.addTableColumnReferences( + accessControl, + session.getIdentity(), + ImmutableMultimap.builder() + .putAll(tableName, metadata.getColumnHandles(session, tableHandle).keySet()) + .build()); + try { + accessControl.checkCanInsertIntoTable(session.getRequiredTransactionId(), session.getIdentity(), tableName); + } + catch (AccessDeniedException exception) { + throw new AccessDeniedException(format("Cannot ANALYZE (missing insert privilege) table %s", tableName)); + } + + analysis.setAnalyzeTarget(tableHandle); + return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT)); + } + @Override protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional scope) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/DistributedExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/DistributedExecutionPlanner.java index c73aa7a211d4..3e511f5189e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/DistributedExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/DistributedExecutionPlanner.java @@ -43,6 +43,7 @@ import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -336,6 +337,12 @@ public Map visitTableFinish(TableFinishNode node, Void return node.getSource().accept(this, context); } + @Override + public Map visitStatisticsWriterNode(StatisticsWriterNode node, Void context) + { + return node.getSource().accept(this, context); + } + @Override public Map visitDelete(DeleteNode node, Void context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index e5fd1f072143..882ba5f22f8b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -68,6 +68,7 @@ import com.facebook.presto.operator.SpatialIndexBuilderOperator.SpatialPredicate; import com.facebook.presto.operator.SpatialJoinOperator.SpatialJoinOperatorFactory; import com.facebook.presto.operator.StageExecutionDescriptor; +import com.facebook.presto.operator.StatisticsWriterOperator.StatisticsWriterOperatorFactory; import com.facebook.presto.operator.StreamingAggregationOperator.StreamingAggregationOperatorFactory; import com.facebook.presto.operator.TableScanOperator.TableScanOperatorFactory; import com.facebook.presto.operator.TaskContext; @@ -150,6 +151,7 @@ import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -2235,6 +2237,22 @@ public PhysicalOperation visitTableWriter(TableWriterNode node, LocalExecutionPl return new PhysicalOperation(operatorFactory, outputMapping.build(), context, source); } + @Override + public PhysicalOperation visitStatisticsWriterNode(StatisticsWriterNode node, LocalExecutionPlanContext context) + { + PhysicalOperation source = node.getSource().accept(this, context); + + StatisticAggregationsDescriptor descriptor = node.getDescriptor().map(symbol -> source.getLayout().get(symbol)); + + OperatorFactory operatorFactory = new StatisticsWriterOperatorFactory( + context.getNextOperatorId(), + node.getId(), + computedStatistics -> metadata.finishStatisticsCollection(session, ((StatisticsWriterNode.WriteStatisticsHandle) node.getTarget()).getHandle(), computedStatistics), + node.isRowCountEnabled(), + descriptor); + return new PhysicalOperation(operatorFactory, makeLayout(node), context, source); + } + @Override public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPlanContext context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java index 39fb55ce18f2..cc79189f82ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java @@ -26,6 +26,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.metadata.QualifiedObjectName; +import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.metadata.TableMetadata; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; @@ -41,6 +42,7 @@ import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; +import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -49,10 +51,13 @@ import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.ValuesNode; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker; +import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.CreateTableAsSelect; import com.facebook.presto.sql.tree.Delete; @@ -67,6 +72,7 @@ import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.Statement; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.AbstractMap.SimpleImmutableEntry; @@ -79,8 +85,10 @@ import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName; import static com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference; import static com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; @@ -195,6 +203,9 @@ private RelationPlan planStatementWithoutOutput(Analysis analysis, Statement sta } return createTableCreationPlan(analysis, ((CreateTableAsSelect) statement).getQuery()); } + else if (statement instanceof Analyze) { + return createAnalyzePlan(analysis, (Analyze) statement); + } else if (statement instanceof Insert) { checkState(analysis.getInsert().isPresent(), "Insert handle is missing"); return createInsertPlan(analysis, (Insert) statement); @@ -223,6 +234,50 @@ private RelationPlan createExplainAnalyzePlan(Analysis analysis, Explain stateme return new RelationPlan(root, scope, ImmutableList.of(outputSymbol)); } + private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) + { + TableHandle targetTable = analysis.getAnalyzeTarget().get(); + + // Plan table scan + Map columnHandles = metadata.getColumnHandles(session, targetTable); + ImmutableList.Builder tableScanOutputs = ImmutableList.builder(); + ImmutableMap.Builder symbolToColumnHandle = ImmutableMap.builder(); + ImmutableMap.Builder columnNameToSymbol = ImmutableMap.builder(); + TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable); + for (ColumnMetadata column : tableMetadata.getColumns()) { + Symbol symbol = symbolAllocator.newSymbol(column.getName(), column.getType()); + tableScanOutputs.add(symbol); + symbolToColumnHandle.put(symbol, columnHandles.get(column.getName())); + columnNameToSymbol.put(column.getName(), symbol); + } + + TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata( + session, + targetTable.getConnectorId().getCatalogName(), + tableMetadata.getMetadata()); + + TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToSymbol.build()); + StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations(); + List groupingSymbols = statisticAggregations.getGroupingSymbols(); + + PlanNode planNode = new StatisticsWriterNode( + idAllocator.getNextId(), + new AggregationNode( + idAllocator.getNextId(), + new TableScanNode(idAllocator.getNextId(), targetTable, tableScanOutputs.build(), symbolToColumnHandle.build()), + statisticAggregations.getAggregations(), + singleGroupingSet(groupingSymbols), + ImmutableList.of(), + AggregationNode.Step.SINGLE, + Optional.empty(), + Optional.empty()), + new StatisticsWriterNode.WriteStatisticsReference(targetTable), + symbolAllocator.newSymbol("rows", BIGINT), + tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), + tableStatisticAggregation.getDescriptor()); + return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputSymbols()); + } + private RelationPlan createTableCreationPlan(Analysis analysis, Query query) { QualifiedObjectName destination = analysis.getCreateTableDestination().get(); @@ -242,7 +297,7 @@ private RelationPlan createTableCreationPlan(Analysis analysis, Query query) .map(ColumnMetadata::getName) .collect(toImmutableList()); - TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadata(session, destination.getCatalogName(), tableMetadata); + TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, destination.getCatalogName(), tableMetadata); return createTableWriterPlan( analysis, @@ -305,7 +360,7 @@ private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) Optional newTableLayout = metadata.getInsertLayout(session, insert.getTarget()); String catalogName = insert.getTarget().getConnectorId().getCatalogName(); - TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadata(session, catalogName, tableMetadata.getMetadata()); + TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata()); return createTableWriterPlan( analysis, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java index 515edca71723..ea9e52595f60 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java @@ -39,6 +39,7 @@ import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -256,6 +257,13 @@ public PlanNode visitExplainAnalyze(ExplainAnalyzeNode node, RewriteContext context) + { + context.get().setCoordinatorOnlyDistribution(); + return context.defaultRewrite(node, context.get()); + } + @Override public PlanNode visitTableFinish(TableFinishNode node, RewriteContext context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java index 40844f6f80ba..79ca1b671d07 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java @@ -54,6 +54,7 @@ import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -589,6 +590,25 @@ public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, Preferred return rebaseAndDeriveProperties(node, child); } + @Override + public PlanWithProperties visitStatisticsWriterNode(StatisticsWriterNode node, PreferredProperties context) + { + PlanWithProperties child = planChild(node, PreferredProperties.any()); + + // if the child is already a gathering exchange, don't add another + if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { + return rebaseAndDeriveProperties(node, child); + } + + if (!child.getProperties().isCoordinatorOnly()) { + child = withDerivedProperties( + gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), + child.getProperties()); + } + + return rebaseAndDeriveProperties(node, child); + } + @Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java index 74e182db6743..f02d171b1d20 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java @@ -46,6 +46,7 @@ import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNNode; @@ -191,6 +192,14 @@ public PlanWithProperties visitSort(SortNode node, StreamPreferredProperties par return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); } + @Override + public PlanWithProperties visitStatisticsWriterNode(StatisticsWriterNode node, StreamPreferredProperties context) + { + // analyze finish requires that all data be in one stream + // this node changes the input organization completely, so we do not pass through parent preferences + return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); + } + @Override public PlanWithProperties visitTableFinish(TableFinishNode node, StreamPreferredProperties parentPreferences) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java index 487c1c1da769..96bfeceee08e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java @@ -32,6 +32,7 @@ import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -116,6 +117,24 @@ public PlanNode visitDelete(DeleteNode node, RewriteContext context) node.getOutputSymbols()); } + @Override + public PlanNode visitStatisticsWriterNode(StatisticsWriterNode node, RewriteContext context) + { + PlanNode child = node.getSource(); + child = child.accept(this, context); + + StatisticsWriterNode.WriteStatisticsHandle analyzeHandle = + new StatisticsWriterNode.WriteStatisticsHandle(metadata.beginStatisticsCollection(session, ((StatisticsWriterNode.WriteStatisticsReference) node.getTarget()).getHandle())); + + return new StatisticsWriterNode( + node.getId(), + child, + analyzeHandle, + node.getRowCountSymbol(), + node.isRowCountEnabled(), + node.getDescriptor()); + } + @Override public PlanNode visitTableFinish(TableFinishNode node, RewriteContext context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java index 23964ab9c5ab..e7ab4934ee4f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java @@ -60,6 +60,7 @@ import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -366,6 +367,14 @@ public ActualProperties visitDistinctLimit(DistinctLimitNode node, List context) + { + return ActualProperties.builder() + .global(coordinatorSingleStreamPartition()) + .build(); + } + @Override public ActualProperties visitTableFinish(TableFinishNode node, List inputProperties) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java index 08b8936656f6..08af492361d6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -51,6 +51,7 @@ import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -653,6 +654,19 @@ public PlanNode visitTableWriter(TableWriterNode node, RewriteContext> context) + { + PlanNode source = context.rewrite(node.getSource(), ImmutableSet.copyOf(node.getSource().getOutputSymbols())); + return new StatisticsWriterNode( + node.getId(), + source, + node.getTarget(), + node.getRowCountSymbol(), + node.isRowCountEnabled(), + node.getDescriptor()); + } + @Override public PlanNode visitTableFinish(TableFinishNode node, RewriteContext> context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java index fbbaa7022d57..ab0651fa102e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java @@ -47,6 +47,7 @@ import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -377,6 +378,14 @@ public StreamProperties visitAggregation(AggregationNode node, List node.getGroupingKeys().contains(symbol) ? Optional.of(symbol) : Optional.empty()); } + @Override + public StreamProperties visitStatisticsWriterNode(StatisticsWriterNode node, List inputProperties) + { + StreamProperties properties = Iterables.getOnlyElement(inputProperties); + // analyze finish only outputs row count + return properties.withUnspecifiedPartitioning(); + } + @Override public StreamProperties visitTableFinish(TableFinishNode node, List inputProperties) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java index d3e1fb758cdf..304ac0ed7788 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNNode; @@ -163,6 +164,17 @@ public TableWriterNode map(TableWriterNode node, PlanNode source, PlanNodeId new node.getStatisticsAggregationDescriptor().map(this::map)); } + public StatisticsWriterNode map(StatisticsWriterNode node, PlanNode source) + { + return new StatisticsWriterNode( + node.getId(), + source, + node.getTarget(), + node.getRowCountSymbol(), + node.isRowCountEnabled(), + node.getDescriptor().map(this::map)); + } + public TableFinishNode map(TableFinishNode node, PlanNode source) { return new TableFinishNode( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index 509fd52cd5d2..1e82d82564a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -54,6 +54,7 @@ import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -361,6 +362,14 @@ public PlanNode visitDelete(DeleteNode node, RewriteContext context) return new DeleteNode(node.getId(), context.rewrite(node.getSource()), node.getTarget(), canonicalize(node.getRowId()), node.getOutputSymbols()); } + @Override + public PlanNode visitStatisticsWriterNode(StatisticsWriterNode node, RewriteContext context) + { + PlanNode source = context.rewrite(node.getSource()); + SymbolMapper mapper = new SymbolMapper(mapping); + return mapper.map(node, source); + } + @Override public PlanNode visitTableFinish(TableFinishNode node, RewriteContext context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanNode.java index 8748c1dce8b0..e6b605a1e4b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanNode.java @@ -61,7 +61,9 @@ @JsonSubTypes.Type(value = ExplainAnalyzeNode.class, name = "explainAnalyze"), @JsonSubTypes.Type(value = ApplyNode.class, name = "apply"), @JsonSubTypes.Type(value = AssignUniqueId.class, name = "assignUniqueId"), - @JsonSubTypes.Type(value = LateralJoinNode.class, name = "lateralJoin")}) + @JsonSubTypes.Type(value = LateralJoinNode.class, name = "lateralJoin"), + @JsonSubTypes.Type(value = StatisticsWriterNode.class, name = "statisticsWriterNode"), +}) public abstract class PlanNode { private final PlanNodeId id; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanVisitor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanVisitor.java index 89e0eb690b49..6ce152c7d995 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanVisitor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/PlanVisitor.java @@ -134,6 +134,11 @@ public R visitTableFinish(TableFinishNode node, C context) return visitPlan(node, context); } + public R visitStatisticsWriterNode(StatisticsWriterNode node, C context) + { + return visitPlan(node, context); + } + public R visitUnion(UnionNode node, C context) { return visitPlan(node, context); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java new file mode 100644 index 000000000000..500693c3cc0e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.plan; + +import com.facebook.presto.metadata.AnalyzeTableHandle; +import com.facebook.presto.metadata.TableHandle; +import com.facebook.presto.sql.planner.Symbol; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class StatisticsWriterNode + extends PlanNode +{ + private final PlanNode source; + private final Symbol rowCountSymbol; + private final WriteStatisticsTarget target; + private final boolean rowCountEnabled; + private final StatisticAggregationsDescriptor descriptor; + + @JsonCreator + public StatisticsWriterNode( + @JsonProperty("id") PlanNodeId id, + @JsonProperty("source") PlanNode source, + @JsonProperty("target") WriteStatisticsTarget target, + @JsonProperty("rowCountSymbol") Symbol rowCountSymbol, + @JsonProperty("rowCountEnabled") boolean rowCountEnabled, + @JsonProperty("descriptor") StatisticAggregationsDescriptor descriptor) + { + super(id); + this.source = requireNonNull(source, "source is null"); + this.target = requireNonNull(target, "target is null"); + this.rowCountSymbol = requireNonNull(rowCountSymbol, "rowCountSymbol is null"); + this.rowCountEnabled = rowCountEnabled; + this.descriptor = requireNonNull(descriptor, "descriptor is null"); + } + + @JsonProperty + public PlanNode getSource() + { + return source; + } + + @JsonProperty + public WriteStatisticsTarget getTarget() + { + return target; + } + + @JsonProperty + public StatisticAggregationsDescriptor getDescriptor() + { + return descriptor; + } + + @JsonProperty + public Symbol getRowCountSymbol() + { + return rowCountSymbol; + } + + @JsonProperty + public boolean isRowCountEnabled() + { + return rowCountEnabled; + } + + @Override + public List getSources() + { + return ImmutableList.of(source); + } + + @Override + public List getOutputSymbols() + { + return ImmutableList.of(rowCountSymbol); + } + + @Override + public PlanNode replaceChildren(List newChildren) + { + return new StatisticsWriterNode( + getId(), + Iterables.getOnlyElement(newChildren), + target, + rowCountSymbol, + rowCountEnabled, + descriptor); + } + + @Override + public R accept(PlanVisitor visitor, C context) + { + return visitor.visitStatisticsWriterNode(this, context); + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "@type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = WriteStatisticsHandle.class, name = "WriteStatisticsHandle")}) + @SuppressWarnings({"EmptyClass", "ClassMayBeInterface"}) + public abstract static class WriteStatisticsTarget + { + @Override + public abstract String toString(); + } + + public static class WriteStatisticsHandle + extends WriteStatisticsTarget + { + private final AnalyzeTableHandle handle; + + @JsonCreator + public WriteStatisticsHandle(@JsonProperty("handle") AnalyzeTableHandle handle) + { + this.handle = requireNonNull(handle, "handle is null"); + } + + @JsonProperty + public AnalyzeTableHandle getHandle() + { + return handle; + } + + @Override + public String toString() + { + return handle.toString(); + } + } + + // only used during planning -- will not be serialized + public static class WriteStatisticsReference + extends WriteStatisticsTarget + { + private final TableHandle handle; + + public WriteStatisticsReference(TableHandle handle) + { + this.handle = requireNonNull(handle, "handle is null"); + } + + public TableHandle getHandle() + { + return handle; + } + + @Override + public String toString() + { + return handle.toString(); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index 565ec02fbf4e..13a094c6f733 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -80,6 +80,7 @@ import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -1121,6 +1122,15 @@ public Void visitTableWriter(TableWriterNode node, Integer indent) return processChildren(node, indent + 1); } + @Override + public Void visitStatisticsWriterNode(StatisticsWriterNode node, Integer indent) + { + print(indent, "- StatisticsWriterNode[%s] => [%s]", node.getTarget(), formatOutputs(node.getOutputSymbols())); + printPlanNodesStatsAndCost(indent + 2, node); + printStats(indent + 2, node.getId()); + return processChildren(node, indent + 1); + } + @Override public Void visitTableFinish(TableFinishNode node, Integer indent) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index 7e8b61df14b1..988d141a209a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -51,6 +51,8 @@ import com.facebook.presto.sql.planner.plan.SetOperationNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -528,6 +530,22 @@ public Void visitMetadataDelete(MetadataDeleteNode node, Set boundSymbol return null; } + @Override + public Void visitStatisticsWriterNode(StatisticsWriterNode node, Set boundSymbols) + { + node.getSource().accept(this, boundSymbols); // visit child + + StatisticAggregationsDescriptor descriptor = node.getDescriptor(); + Set dependencies = ImmutableSet.builder() + .addAll(descriptor.getGrouping().values()) + .addAll(descriptor.getColumnStatistics().values()) + .addAll(descriptor.getTableStatistics().values()) + .build(); + List outputSymbols = node.getSource().getOutputSymbols(); + checkDependencies(dependencies, dependencies, "Invalid node. Dependencies (%s) not in source plan output (%s)", dependencies, outputSymbols); + return null; + } + @Override public Void visitTableFinish(TableFinishNode node, Set boundSymbols) { diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 60c9cf8c1613..8acf6e7a381c 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -65,6 +65,7 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.index.IndexManager; import com.facebook.presto.memory.MemoryManagerConfig; +import com.facebook.presto.metadata.AnalyzePropertyManager; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; import com.facebook.presto.metadata.HandleResolver; @@ -304,6 +305,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); this.planFragmenter = new PlanFragmenter(this.metadata, this.nodePartitioningManager, new QueryManagerConfig()); this.joinCompiler = new JoinCompiler(metadata, featuresConfig); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java index 347d937279c1..d06ec7f879ae 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java @@ -85,6 +85,12 @@ public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTable return new TestingTableHandle(tableName); } + @Override + public ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map analyzeProperties) + { + return getTableHandle(session, tableName); + } + @Override public List getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint constraint, Optional> desiredColumns) { diff --git a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index f9135dcff248..cc28608626ff 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -43,6 +43,7 @@ import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -96,7 +97,8 @@ private enum NodeType TABLE_WRITER, TABLE_FINISH, INDEX_SOURCE, - UNNEST + UNNEST, + ANALYZE_FINISH, } private static final Map NODE_COLORS = immutableEnumMap(ImmutableMap.builder() @@ -120,6 +122,7 @@ private enum NodeType .put(NodeType.INDEX_SOURCE, "dodgerblue3") .put(NodeType.UNNEST, "crimson") .put(NodeType.SAMPLE, "goldenrod4") + .put(NodeType.ANALYZE_FINISH, "plum") .build()); static { @@ -224,6 +227,13 @@ public Void visitTableWriter(TableWriterNode node, Void context) return node.getSource().accept(this, context); } + @Override + public Void visitStatisticsWriterNode(StatisticsWriterNode node, Void context) + { + printNode(node, format("StatisticsWriterNode[%s]", Joiner.on(", ").join(node.getOutputSymbols())), NODE_COLORS.get(NodeType.ANALYZE_FINISH)); + return node.getSource().accept(this, context); + } + @Override public Void visitTableFinish(TableFinishNode node, Void context) { diff --git a/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java b/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java index c57c6ff9fa7f..97c13c2016cf 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.resourceGroups.QueryType; import com.facebook.presto.sql.tree.AddColumn; +import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateSchema; @@ -70,6 +71,7 @@ private StatementUtils() {} builder.put(Query.class, QueryType.SELECT); builder.put(Explain.class, QueryType.EXPLAIN); + builder.put(Analyze.class, QueryType.ANALYZE); builder.put(CreateTableAsSelect.class, QueryType.INSERT); builder.put(Insert.class, QueryType.INSERT); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java index 1e6930e9c79c..7fd434e35ed6 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.AnalyzePropertyManager; import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; @@ -72,6 +73,7 @@ public TestResetSessionTask() new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); metadata.getSessionPropertyManager().addSystemSessionProperty(stringProperty( diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java index ccd17875fe41..7f27a60d9ee2 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java @@ -15,6 +15,7 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.AnalyzePropertyManager; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; import com.facebook.presto.metadata.MetadataManager; @@ -70,6 +71,7 @@ public TestSetPathTask() new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java index 93c347d5e587..ce0837dc5112 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java @@ -15,6 +15,7 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.AnalyzePropertyManager; import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; @@ -84,6 +85,7 @@ public TestSetSessionTask() new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); metadata.getSessionPropertyManager().addSystemSessionProperty(stringProperty( diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index fcdbabd6b04d..f277b83f8174 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -99,6 +99,12 @@ public Optional getTableHandle(Session session, QualifiedObjectName throw new UnsupportedOperationException(); } + @Override + public Optional getTableHandleForStatisticsCollection(Session session, QualifiedObjectName tableName, Map analyzeProperties) + { + throw new UnsupportedOperationException(); + } + @Override public Optional getSystemTable(Session session, QualifiedObjectName tableName) { @@ -243,12 +249,30 @@ public Optional getInsertLayout(Session session, TableHandle tar throw new UnsupportedOperationException(); } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, String catalogName, ConnectorTableMetadata tableMetadata) + { + throw new UnsupportedOperationException(); + } + @Override public TableStatisticsMetadata getStatisticsCollectionMetadata(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { throw new UnsupportedOperationException(); } + @Override + public AnalyzeTableHandle beginStatisticsCollection(Session session, TableHandle tableHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public void finishStatisticsCollection(Session session, AnalyzeTableHandle tableHandle, Collection computedStatistics) + { + throw new UnsupportedOperationException(); + } + @Override public void beginQuery(Session session, Set connectors) { @@ -417,6 +441,12 @@ public ColumnPropertyManager getColumnPropertyManager() throw new UnsupportedOperationException(); } + @Override + public AnalyzePropertyManager getAnalyzePropertyManager() + { + throw new UnsupportedOperationException(); + } + @Override public void dropColumn(Session session, TableHandle tableHandle, ColumnHandle column) { diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java index 8cf42157b975..2f8ac2ad5127 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java @@ -99,6 +99,7 @@ public TestInformationSchemaMetadata() new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index 81a7dbee7b75..dfa3d9f03b71 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -23,6 +23,7 @@ import com.facebook.presto.execution.TaskManagerConfig; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.memory.MemoryManagerConfig; +import com.facebook.presto.metadata.AnalyzePropertyManager; import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; @@ -45,6 +46,7 @@ import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.TypeManager; @@ -60,6 +62,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.List; import java.util.Optional; import java.util.function.Consumer; @@ -67,6 +70,8 @@ import static com.facebook.presto.connector.ConnectorId.createSystemTablesConnectorId; import static com.facebook.presto.metadata.ViewDefinition.ViewColumn; import static com.facebook.presto.operator.scalar.ApplyFunction.APPLY_FUNCTION; +import static com.facebook.presto.spi.session.PropertyMetadata.integerProperty; +import static com.facebook.presto.spi.session.PropertyMetadata.stringProperty; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; @@ -1096,6 +1101,16 @@ public void testCreateTable() assertFails(DUPLICATE_PROPERTY, ".* Duplicate property: p1", "CREATE TABLE test (id bigint) WITH (p1 = 'p1', \"p1\" = 'p2')"); } + @Test + public void testAnalyze() + { + analyze("ANALYZE t1"); + analyze("ANALYZE t1 WITH (p1 = 'p1')"); + + assertFails(DUPLICATE_PROPERTY, ".* Duplicate property: p1", "ANALYZE t1 WITH (p1 = 'p1', p2 = 2, p1 = 'p3')"); + assertFails(DUPLICATE_PROPERTY, ".* Duplicate property: p1", "ANALYZE t1 WITH (p1 = 'p1', \"p1\" = 'p2')"); + } + @Test public void testCreateSchema() { @@ -1500,11 +1515,15 @@ public void setup() new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), + new AnalyzePropertyManager(), transactionManager); metadata.getFunctionRegistry().addFunctions(ImmutableList.of(APPLY_FUNCTION)); - catalogManager.registerCatalog(createTestingCatalog(TPCH_CATALOG, TPCH_CONNECTOR_ID)); + Catalog tpchTestCatalog = createTestingCatalog(TPCH_CATALOG, TPCH_CONNECTOR_ID); + catalogManager.registerCatalog(tpchTestCatalog); + metadata.getAnalyzePropertyManager().addProperties(TPCH_CONNECTOR_ID, tpchTestCatalog.getConnector(TPCH_CONNECTOR_ID).getAnalyzeProperties()); + catalogManager.registerCatalog(createTestingCatalog(SECOND_CATALOG, SECOND_CONNECTOR_ID)); catalogManager.registerCatalog(createTestingCatalog(THIRD_CATALOG, THIRD_CONNECTOR_ID)); @@ -1763,6 +1782,14 @@ public ConnectorSplitManager getSplitManager() { throw new UnsupportedOperationException(); } + + @Override + public List> getAnalyzeProperties() + { + return ImmutableList.of( + stringProperty("p1", "test string property", "", false), + integerProperty("p2", "test integer property", 0, false)); + } }; } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index 5a621f4a6e93..34afcc110107 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -31,6 +31,7 @@ import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.plan.ValuesNode; import com.facebook.presto.sql.tree.LongLiteral; @@ -108,6 +109,21 @@ public class TestLogicalPlanner { // TODO: Use com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder#tableScan with required node/stream // partitioning to properly test aggregation, window function and join. + @Test + public void testAnalyze() + { + assertDistributedPlan("ANALYZE orders", + anyTree( + node(StatisticsWriterNode.class, + anyTree( + exchange(REMOTE, GATHER, + node(AggregationNode.class, + anyTree( + exchange(REMOTE, GATHER, + node(AggregationNode.class, + tableScan("orders", ImmutableMap.of())))))))))); + } + @Test public void testAggregation() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java index dad56cb675b6..a5aee87332bd 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java @@ -62,6 +62,7 @@ public enum StandardErrorCode INVALID_COLUMN_PROPERTY(0x0000_0027, USER_ERROR), QUERY_HAS_TOO_MANY_STAGES(0x0000_0028, USER_ERROR), INVALID_SPATIAL_PARTITIONING(0x0000_0029, USER_ERROR), + INVALID_ANALYZE_PROPERTY(0x0000_002A, USER_ERROR), GENERIC_INTERNAL_ERROR(0x0001_0000, INTERNAL_ERROR), TOO_MANY_REQUESTS_FAILED(0x0001_0001, INTERNAL_ERROR), diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java index 0e2339147db5..5b61e1c6e880 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java @@ -108,6 +108,14 @@ default List> getSchemaProperties() return emptyList(); } + /** + * @return the analyze properties for this connector + */ + default List> getAnalyzeProperties() + { + return emptyList(); + } + /** * @return the table properties for this connector */ diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAnalyzeMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAnalyzeMetadata.java new file mode 100644 index 000000000000..45ecfa3b3f43 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAnalyzeMetadata.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.connector; + +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.statistics.TableStatisticsMetadata; + +import static java.util.Objects.requireNonNull; + +public class ConnectorAnalyzeMetadata +{ + private final TableStatisticsMetadata statisticsMetadata; + private final ConnectorTableHandle tableHandle; + + public ConnectorAnalyzeMetadata(TableStatisticsMetadata statisticsMetadata, ConnectorTableHandle tableHandle) + { + this.statisticsMetadata = requireNonNull(statisticsMetadata, "statisticsMetadata is null"); + this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); + } + + public TableStatisticsMetadata getStatisticsMetadata() + { + return statisticsMetadata; + } + + public ConnectorTableHandle getTableHandle() + { + return tableHandle; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java index d8d21f68e720..298426ade777 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java @@ -74,6 +74,15 @@ default boolean schemaExists(ConnectorSession session, String schemaName) */ ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName); + /** + * Returns a table handle for the specified table name, or null if the connector does not contain the table. + * The returned table handle can contain information in analyzeProperties. + */ + default ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map analyzeProperties) + { + throw new PrestoException(NOT_SUPPORTED, "This connector does not support analyze"); + } + /** * Returns the system table for the specified table name, if one exists. * The system tables handled via {@link #getSystemTable} differ form those returned by {@link Connector#getSystemTables()}. @@ -303,11 +312,35 @@ default Optional getInsertLayout(ConnectorSession sessi /** * Describes statistics that must be collected during a write. */ - default TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) + default TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) { return TableStatisticsMetadata.empty(); } + /** + * Describe statistics that must be collected during a statistics collection + */ + default TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) + { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata getTableHandleForStatisticsCollection() is implemented without getStatisticsCollectionMetadata()"); + } + + /** + * Begin statistics collection + */ + default ConnectorTableHandle beginStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle) + { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata getStatisticsCollectionMetadata() is implemented without beginStatisticsCollection()"); + } + + /** + * Finish statistics collection + */ + default void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle, Collection computedStatistics) + { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata beginStatisticsCollection() is implemented without finishStatisticsCollection()"); + } + /** * Begin the atomic creation of a table with data. */ diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java index df571bed8ec8..3c240576eab5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java @@ -115,6 +115,14 @@ public Optional getInsertLayout(ConnectorSession sessio } } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.getStatisticsCollectionMetadataForWrite(session, tableMetadata); + } + } + @Override public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) { @@ -123,6 +131,22 @@ public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession } } + @Override + public ConnectorTableHandle beginStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.beginStatisticsCollection(session, tableHandle); + } + } + + @Override + public void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle, Collection computedStatistics) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + delegate.finishStatisticsCollection(session, tableHandle, computedStatistics); + } + } + @Override public boolean schemaExists(ConnectorSession session, String schemaName) { @@ -147,6 +171,14 @@ public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTable } } + @Override + public ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map analyzeProperties) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.getTableHandleForStatisticsCollection(session, tableName, analyzeProperties); + } + } + @Override public Optional getSystemTable(ConnectorSession session, SchemaTableName tableName) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/resourceGroups/QueryType.java b/presto-spi/src/main/java/com/facebook/presto/spi/resourceGroups/QueryType.java index f037e70d6186..f577ad17d538 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/resourceGroups/QueryType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/resourceGroups/QueryType.java @@ -19,6 +19,7 @@ public enum QueryType DELETE, DESCRIBE, EXPLAIN, + ANALYZE, INSERT, SELECT } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueryRunner.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueryRunner.java index 558d4193f87d..f3b0e497a2f1 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueryRunner.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueryRunner.java @@ -15,6 +15,10 @@ import org.testng.annotations.Test; +import static com.facebook.presto.testing.TestingAccessControlManager.TestingPrivilegeType.INSERT_TABLE; +import static com.facebook.presto.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN; +import static com.facebook.presto.testing.TestingAccessControlManager.privilege; + public class TestLocalQueryRunner extends AbstractTestQueryFramework { @@ -28,4 +32,13 @@ public void testSimpleQuery() { assertQuery("SELECT * FROM nation"); } + + @Test + public void testAnalyzeAccessControl() + { + assertAccessAllowed("ANALYZE nation"); + assertAccessDenied("ANALYZE nation", "Cannot ANALYZE \\(missing insert privilege\\) table .*.nation.*", privilege("nation", INSERT_TABLE)); + assertAccessDenied("ANALYZE nation", "Cannot select from columns \\[.*] in table or view .*.nation", privilege("nation", SELECT_COLUMN)); + assertAccessDenied("ANALYZE nation", "Cannot select from columns \\[.*nationkey.*] in table or view .*.nation", privilege("nationkey", SELECT_COLUMN)); + } } diff --git a/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java b/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java index a4875bcac356..07a64728193c 100644 --- a/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java +++ b/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java @@ -33,9 +33,11 @@ import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.statistics.ColumnStatistics; +import com.facebook.presto.spi.statistics.ComputedStatistics; import com.facebook.presto.spi.statistics.DoubleRange; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.spi.statistics.TableStatisticsMetadata; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; import com.facebook.presto.tpch.statistics.ColumnStatisticsData; @@ -61,12 +63,14 @@ import io.airlift.tpch.TpchTable; import java.time.LocalDate; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; @@ -172,6 +176,12 @@ public TpchTableHandle getTableHandle(ConnectorSession session, SchemaTableName return new TpchTableHandle(tableName.getTableName(), scaleFactor); } + @Override + public ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map analyzeProperties) + { + return getTableHandle(session, tableName); + } + @Override public List getTableLayouts( ConnectorSession session, @@ -412,6 +422,24 @@ private static double toDouble(Object value, Type columnType) throw new IllegalArgumentException("unsupported column type " + columnType); } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) + { + return new TableStatisticsMetadata(ImmutableSet.of(), ImmutableSet.of(ROW_COUNT), ImmutableList.of()); + } + + @Override + public ConnectorTableHandle beginStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle) + { + return (TpchTableHandle) tableHandle; + } + + @Override + public void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle, Collection computedStatistics) + { + // do nothing + } + @VisibleForTesting TpchColumnHandle toColumnHandle(TpchColumn column) {