diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index b071a6ec5c25b..ab28961632fee 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -237,6 +237,7 @@ public final class SystemSessionProperties public static final String EXCEEDED_MEMORY_LIMIT_HEAP_DUMP_FILE_DIRECTORY = "exceeded_memory_limit_heap_dump_file_directory"; public static final String DISTRIBUTED_TRACING_MODE = "distributed_tracing_mode"; public static final String VERBOSE_RUNTIME_STATS_ENABLED = "verbose_runtime_stats_enabled"; + public static final String VERBOSE_OPTIMIZER_INFO_ENABLED = "verbose_optimizer_info_enabled"; public static final String STREAMING_FOR_PARTIAL_AGGREGATION_ENABLED = "streaming_for_partial_aggregation_enabled"; public static final String MAX_STAGE_COUNT_FOR_EAGER_SCHEDULING = "max_stage_count_for_eager_scheduling"; public static final String HYPERLOGLOG_STANDARD_ERROR_WARNING_THRESHOLD = "hyperloglog_standard_error_warning_threshold"; @@ -1292,6 +1293,11 @@ public SystemSessionProperties( "Enable logging all runtime stats", featuresConfig.isVerboseRuntimeStatsEnabled(), false), + booleanProperty( + VERBOSE_OPTIMIZER_INFO_ENABLED, + "Enable logging of verbose information about applied optimizations", + featuresConfig.isVerboseOptimizerInfoEnabled(), + false), booleanProperty( STREAMING_FOR_PARTIAL_AGGREGATION_ENABLED, "Enable streaming for partial aggregation", @@ -2460,6 +2466,11 @@ public static boolean isVerboseRuntimeStatsEnabled(Session session) return session.getSystemProperty(VERBOSE_RUNTIME_STATS_ENABLED, Boolean.class); } + public static boolean isVerboseOptimizerInfoEnabled(Session session) + { + return session.getSystemProperty(VERBOSE_OPTIMIZER_INFO_ENABLED, Boolean.class); + } + public static boolean isLeafNodeLimitEnabled(Session session) { return session.getSystemProperty(LEAF_NODE_LIMIT_ENABLED, Boolean.class); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java index e57950a740271..543e79fcbae0f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java @@ -26,11 +26,13 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.eventlistener.PlanOptimizerInformation; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.plan.JoinNode; @@ -42,6 +44,7 @@ import static com.facebook.presto.SystemSessionProperties.getQueryAnalyzerTimeout; import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static com.facebook.presto.common.RuntimeUnit.NANO; import static com.facebook.presto.spi.StandardErrorCode.QUERY_PLANNING_TIMEOUT; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; @@ -105,11 +108,13 @@ public Plan validateAndOptimizePlan(PlanNode root, PlanStage stage) throw new PrestoException(QUERY_PLANNING_TIMEOUT, String.format("The query optimizer exceeded the timeout of %s.", getQueryAnalyzerTimeout(session).toString())); } long start = System.nanoTime(); - root = optimizer.optimize(root, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector); - requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName())); + PlanNode newRoot = optimizer.optimize(root, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector); + requireNonNull(newRoot, format("%s returned a null plan", optimizer.getClass().getName())); if (enableVerboseRuntimeStats) { session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", optimizer.getClass().getSimpleName()), NANO, System.nanoTime() - start); } + collectOptimizerInformation(optimizer, root, newRoot); + root = newRoot; } } @@ -133,4 +138,22 @@ private StatsAndCosts computeStats(PlanNode root, TypeProvider types) } return StatsAndCosts.empty(); } + + private void collectOptimizerInformation(PlanOptimizer optimizer, PlanNode oldNode, PlanNode newNode) + { + if (optimizer instanceof IterativeOptimizer) { + // iterative optimizers do their own recording of what rules got triggered + return; + } + + boolean isTriggered = (oldNode != newNode); + boolean isApplicable = + isTriggered || + !optimizer.isEnabled(session) && isVerboseOptimizerInfoEnabled(session) && + optimizer.isApplicable(oldNode, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector); + + if (isTriggered || isApplicable) { + session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(optimizer.getClass().getSimpleName(), isTriggered, Optional.of(isApplicable))); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 27107700e5f42..aedd0561cf89a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -234,6 +234,7 @@ public class FeaturesConfig private AggregationIfToFilterRewriteStrategy aggregationIfToFilterRewriteStrategy = AggregationIfToFilterRewriteStrategy.DISABLED; private String analyzerType = "BUILTIN"; private boolean verboseRuntimeStatsEnabled; + private boolean verboseOptimizerInfoEnabled; private boolean streamingForPartialAggregationEnabled; private boolean preferMergeJoinForSortedInputs; @@ -2205,6 +2206,11 @@ public boolean isVerboseRuntimeStatsEnabled() return verboseRuntimeStatsEnabled; } + public boolean isVerboseOptimizerInfoEnabled() + { + return verboseOptimizerInfoEnabled; + } + @Config("verbose-runtime-stats-enabled") @ConfigDescription("Enable logging all runtime stats.") public FeaturesConfig setVerboseRuntimeStatsEnabled(boolean value) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java index dac87764252e9..027a56f2c4fc4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java @@ -43,6 +43,7 @@ import java.util.Set; import java.util.stream.Stream; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static com.facebook.presto.common.RuntimeUnit.NANO; import static com.facebook.presto.spi.StandardErrorCode.OPTIMIZER_TIMEOUT; import static com.google.common.base.Preconditions.checkArgument; @@ -167,6 +168,9 @@ private boolean exploreNode(int group, Context context, Matcher matcher) Rule rule = possiblyMatchingRules.next(); if (!rule.isEnabled(context.session)) { + if (isVerboseOptimizerInfoEnabled(context.session) && isApplicable(node, rule, matcher, context)) { + context.addRulesApplicable(rule.getClass().getSimpleName()); + } continue; } @@ -224,6 +228,17 @@ private Rule.Result transform(PlanNode node, Rule rule, Matcher matcher, return result; } + private boolean isApplicable(PlanNode node, Rule rule, Matcher matcher, Context context) + { + Match match = matcher.match(rule.getPattern(), node); + if (match.isEmpty()) { + return false; + } + + Rule.Result result = rule.apply(match.value(), match.captures(), ruleContext(context)); + return !result.isEmpty(); + } + private boolean exploreChildren(int group, Context context, Matcher matcher) { boolean progress = false; @@ -313,6 +328,7 @@ private static class Context private final CostProvider costProvider; private final StatsProvider statsProvider; private final Set rulesTriggered; + private final Set rulesApplicable; public Context( Memo memo, @@ -339,6 +355,7 @@ public Context( this.costProvider = costProvider; this.statsProvider = statsProvider; this.rulesTriggered = new HashSet<>(); + this.rulesApplicable = new HashSet<>(); } public void checkTimeoutNotExhausted() @@ -353,9 +370,15 @@ public void addRulesTriggered(String rule) rulesTriggered.add(rule); } + public void addRulesApplicable(String rule) + { + rulesApplicable.add(rule); + } + public void collectOptimizerInformation() { rulesTriggered.forEach(x -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(x, true, Optional.empty()))); + rulesApplicable.forEach(x -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(x, false, Optional.of(true)))); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java index cab0fcd74d49a..e916d8e6a92be 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java @@ -93,12 +93,25 @@ public class HashGenerationOptimizer implements PlanOptimizer { private final FunctionAndTypeManager functionAndTypeManager; + private boolean isEnabledForTesting; public HashGenerationOptimizer(FunctionAndTypeManager functionAndTypeManager) { this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionManager is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || SystemSessionProperties.isOptimizeHashGenerationEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { @@ -107,7 +120,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var requireNonNull(types, "types is null"); requireNonNull(variableAllocator, "variableAllocator is null"); requireNonNull(idAllocator, "idAllocator is null"); - if (SystemSessionProperties.isOptimizeHashGenerationEnabled(session)) { + if (isEnabled(session)) { PlanWithProperties result = new Rewriter(idAllocator, variableAllocator, functionAndTypeManager).accept(plan, new HashComputationSet()); return result.getNode(); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java index 4d2084f3d8355..9a34c9bac2924 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java @@ -44,12 +44,25 @@ public class HistoricalStatisticsEquivalentPlanMarkingOptimizer private static final Set> LIMITING_NODES = ImmutableSet.of(TopNNode.class, LimitNode.class, DistinctLimitNode.class, TopNRowNumberNode.class); private final StatsCalculator statsCalculator; + private boolean isEnabledForTesting; public HistoricalStatisticsEquivalentPlanMarkingOptimizer(StatsCalculator statsCalculator) { this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || useHistoryBasedPlanStatisticsEnabled(session) || trackHistoryBasedPlanStatisticsEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { @@ -59,7 +72,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var requireNonNull(variableAllocator, "variableAllocator is null"); requireNonNull(idAllocator, "idAllocator is null"); - if (!useHistoryBasedPlanStatisticsEnabled(session) && !trackHistoryBasedPlanStatisticsEnabled(session)) { + if (!isEnabled(session)) { return plan; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java index a1013196d42f6..3dc1f8104b186 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java @@ -68,23 +68,39 @@ public class KeyBasedSampler implements PlanOptimizer { private final Metadata metadata; + private boolean isEnabledForTesting; public KeyBasedSampler(Metadata metadata, SqlParser sqlParser) { this.metadata = requireNonNull(metadata, "metadata is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isKeyBasedSamplingEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (isKeyBasedSamplingEnabled(session)) { + if (isEnabled(session)) { List sampledFields = new ArrayList<>(2); PlanNode rewritten = SimplePlanRewriter.rewriteWith(new Rewriter(session, metadata.getFunctionAndTypeManager(), idAllocator, sampledFields), plan, null); - if (!sampledFields.isEmpty()) { - warningCollector.add(new PrestoWarning(SAMPLED_FIELDS, String.format("Sampled the following columns/derived columns at %s percent:%n\t%s", getKeyBasedSamplingPercentage(session) * 100., String.join("\n\t", sampledFields)))); - } - else { - warningCollector.add(new PrestoWarning(SEMANTIC_WARNING, "Sampling could not be performed due to the query structure")); + + if (!isEnabledForTesting) { + if (!sampledFields.isEmpty()) { + warningCollector.add(new PrestoWarning(SAMPLED_FIELDS, String.format("Sampled the following columns/derived columns at %s percent:%n\t%s", getKeyBasedSamplingPercentage(session) * 100., String.join("\n\t", sampledFields)))); + } + else { + warningCollector.add(new PrestoWarning(SEMANTIC_WARNING, "Sampling could not be performed due to the query structure")); + } } return rewritten; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java index 90fa34e7ac247..01bfbce2464b9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java @@ -40,6 +40,7 @@ public class MergeJoinForSortedInputOptimizer { private final Metadata metadata; private final SqlParser parser; + private boolean isEnabledForTesting; public MergeJoinForSortedInputOptimizer(Metadata metadata, SqlParser parser) { @@ -47,6 +48,18 @@ public MergeJoinForSortedInputOptimizer(Metadata metadata, SqlParser parser) this.parser = requireNonNull(parser, "parser is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isGroupedExecutionEnabled(session) && preferMergeJoinForSortedInputs(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider type, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { @@ -55,7 +68,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider type, Vari requireNonNull(variableAllocator, "variableAllocator is null"); requireNonNull(idAllocator, "idAllocator is null"); - if (isGroupedExecutionEnabled(session) && preferMergeJoinForSortedInputs(session)) { + if (isEnabled(session)) { return SimplePlanRewriter.rewriteWith(new MergeJoinForSortedInputOptimizer.Rewriter(variableAllocator, idAllocator, metadata, session), plan, null); } return plan; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergePartialAggregationsWithFilter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergePartialAggregationsWithFilter.java index 081312bb5a5ce..82a36f049f759 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergePartialAggregationsWithFilter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergePartialAggregationsWithFilter.java @@ -90,16 +90,29 @@ public class MergePartialAggregationsWithFilter implements PlanOptimizer { private final FunctionAndTypeManager functionAndTypeManager; + private boolean isEnabledForTesting; public MergePartialAggregationsWithFilter(FunctionAndTypeManager functionAndTypeManager) { this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isMergeAggregationsWithAndWithoutFilter(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (isMergeAggregationsWithAndWithoutFilter(session)) { + if (isEnabled(session)) { return SimplePlanRewriter.rewriteWith(new Rewriter(session, variableAllocator, idAllocator, functionAndTypeManager), plan, new Context()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java index efb1e4f87859e..bbd0408b011a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java @@ -83,6 +83,7 @@ public class OptimizeMixedDistinctAggregations { private final Metadata metadata; private final StandardFunctionResolution functionResolution; + private boolean isEnabledForTesting; public OptimizeMixedDistinctAggregations(Metadata metadata) { @@ -90,10 +91,22 @@ public OptimizeMixedDistinctAggregations(Metadata metadata) this.functionResolution = new FunctionResolution(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver()); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isOptimizeDistinctAggregationEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (isOptimizeDistinctAggregationEnabled(session)) { + if (isEnabled(session)) { return SimplePlanRewriter.rewriteWith(new Optimizer(idAllocator, variableAllocator, metadata, functionResolution), plan, Optional.empty()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java index 372e28d4ea241..d56e1e8709491 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java @@ -122,6 +122,7 @@ public class PayloadJoinOptimizer implements PlanOptimizer { private final Metadata metadata; + private boolean isEnabledForTesting; public PayloadJoinOptimizer(Metadata metadata) { @@ -141,9 +142,16 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var return plan; } - private boolean isEnabled(Session session) + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) { - return isOptimizePayloadJoins(session); + return isEnabledForTesting || isOptimizePayloadJoins(session); } private static class Rewriter diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanOptimizer.java index a66e9449b4f2b..752cc5c9165bd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanOptimizer.java @@ -28,4 +28,35 @@ PlanNode optimize(PlanNode plan, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector); + + default boolean isEnabled(Session session) + { + return true; + } + + default void setEnabledForTesting(boolean isSet) + { + return; + } + + default boolean isApplicable(PlanNode plan, + Session session, + TypeProvider types, + VariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator, + WarningCollector warningCollector) + { + setEnabledForTesting(true); + + boolean isApplicable = false; + try { + // wrap in try/catch block in case optimization throws an error + PlanNode newPlan = optimize(plan, session, types, variableAllocator, idAllocator, warningCollector); + isApplicable = !plan.equals(newPlan); + } + finally { + setEnabledForTesting(false); + return isApplicable; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java index 3ee62df7a79b3..0dfa3015a9977 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java @@ -89,12 +89,26 @@ public class PrefilterForLimitingAggregation { private final Metadata metadata; private final StatsCalculator statsCalculator; + private boolean isEnabledForTesting; + public PrefilterForLimitingAggregation(Metadata metadata, StatsCalculator statsCalculator) { this.metadata = metadata; this.statsCalculator = statsCalculator; } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || SystemSessionProperties.isPrefilterForGroupbyLimit(session); + } + @Override public PlanNode optimize( PlanNode plan, @@ -104,7 +118,7 @@ public PlanNode optimize( PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (SystemSessionProperties.isPrefilterForGroupbyLimit(session)) { + if (isEnabled(session)) { return SimplePlanRewriter.rewriteWith(new Rewriter(session, metadata, types, statsCalculator, idAllocator, variableAllocator), plan); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java.orig b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java.orig new file mode 100644 index 0000000000000..2fb0e65d29dc0 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java.orig @@ -0,0 +1,268 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.cost.StatsCalculator; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; +import com.facebook.presto.sql.planner.plan.SortNode; +import com.facebook.presto.sql.tree.Join; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.facebook.presto.common.function.OperatorType.EQUAL; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IF; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.planner.PlannerUtils.addAggregation; +import static com.facebook.presto.sql.planner.PlannerUtils.addProjections; +import static com.facebook.presto.sql.planner.PlannerUtils.clonePlanNode; +import static com.facebook.presto.sql.planner.PlannerUtils.createMapType; +import static com.facebook.presto.sql.planner.PlannerUtils.getHashExpression; +import static com.facebook.presto.sql.planner.PlannerUtils.getTableScanNodeWithOnlyFilterAndProject; +import static com.facebook.presto.sql.planner.PlannerUtils.projectExpressions; +import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.isAllLowCardinalityGroupByKeys; +import static com.facebook.presto.sql.planner.optimizations.JoinNodeUtils.typeConvert; +import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; +import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static java.lang.Boolean.TRUE; + +/** + * An optimization for quicker execution of simple group by + limit queres. In SQL terms, it will be: + * + * Original: + * + * SELECT SUM(x), userid FROM Table GROUP BY userid LIMIT 1000 + * + * Rewritten: + * + * SELECT SUM(x) , userid FROM Table + * CROSS JOIN (SELECT MAP_AGG(hash(userid)) m FROM (SELECT DISTINCT userid FROM Table LIMIT 1000))) + * WHERE IF(CARDINALITY(m)=1000, m[hash(userid)], TRUE) + * + * In addition we also add a timeout to the distinctlimit we add so that we don't get stuck trying to find the keys + */ + +public class PrefilterForLimitingAggregation + implements PlanOptimizer +{ + private final Metadata metadata; + private final StatsCalculator statsCalculator; + public PrefilterForLimitingAggregation(Metadata metadata, StatsCalculator statsCalculator) + { + this.metadata = metadata; + this.statsCalculator = statsCalculator; + } + + @Override + public PlanNode optimize( + PlanNode plan, + Session session, + TypeProvider types, + VariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator, + WarningCollector warningCollector) + { + if (SystemSessionProperties.isPrefilterForGroupbyLimit(session)) { + return SimplePlanRewriter.rewriteWith(new Rewriter(session, metadata, types, statsCalculator, idAllocator, variableAllocator), plan); + } + + return plan; + } + + private static class Rewriter + extends SimplePlanRewriter + { + private final Session session; + private final Metadata metadata; + private final TypeProvider types; + private final StatsCalculator statsCalculator; + private final PlanNodeIdAllocator idAllocator; + private final VariableAllocator variableAllocator; + + private Rewriter( + Session session, + Metadata metadata, + TypeProvider types, + StatsCalculator statsCalculator, + PlanNodeIdAllocator idAllocator, + VariableAllocator variableAllocator) + { + this.session = session; + this.metadata = metadata; + this.types = types; + this.statsCalculator = statsCalculator; + this.idAllocator = idAllocator; + this.variableAllocator = variableAllocator; + } + + @Override + public PlanNode visitSort(SortNode sortNode, RewriteContext context) + { + return sortNode; + } + + @Override + public PlanNode visitLimit(LimitNode limitNode, RewriteContext context) + { + PlanNode source = rewriteWith(this, limitNode.getSource()); + AggregationNode aggregationNode = null; + + if (source instanceof ProjectNode && ((ProjectNode) source).getSource() instanceof AggregationNode) { + aggregationNode = (AggregationNode) ((ProjectNode) source).getSource(); + } + else if (source instanceof AggregationNode) { + aggregationNode = (AggregationNode) source; + } + + if (aggregationNode != null && + !aggregationNode.getGroupingKeys().isEmpty()) { + Optional scanNode = getTableScanNodeWithOnlyFilterAndProject(aggregationNode.getSource()); + // Since we duplicate the source of the aggregation - we want to restrict it to simple scan/filter/project + // so we can do this opportunistic optimization without too much latency/cpu overhead to support common BI usecases + if (scanNode.isPresent() && + !isAllLowCardinalityGroupByKeys(aggregationNode, scanNode.get(), session, statsCalculator, types, limitNode.getCount())) { + PlanNode rewrittenAggregation = addPrefilter(aggregationNode, limitNode.getCount()); + if (rewrittenAggregation != aggregationNode) { + if (source == aggregationNode) { + return replaceChildren(limitNode, ImmutableList.of(rewrittenAggregation)); + } + + return replaceChildren(limitNode, ImmutableList.of(replaceChildren(source, ImmutableList.of(rewrittenAggregation)))); + } + } + } + + if (source == limitNode.getSource()) { + return replaceChildren(limitNode, ImmutableList.of(source)); + } + + return limitNode; + } + + private PlanNode addPrefilter(AggregationNode aggregationNode, long count) + { + List keys = aggregationNode.getGroupingKeys().stream().collect(Collectors.toList()); + if (keys.isEmpty()) { + return aggregationNode; + } + + PlanNode originalSource = aggregationNode.getSource(); +<<<<<<< HEAD + PlanNode keySource = clonePlanNode(originalSource, session, metadata, idAllocator, keys, ImmutableMap.of()); + // TODO(kaikalur): See if timetout can be done in a cleaner way in the middle tier +======= + PlanNode keySource = clonePlanNode(originalSource, session, metadata, idAllocator, keys, new HashMap<>()); +>>>>>>> efe19190ab (Optimize joins over tables with map columns) + DistinctLimitNode timedDistinctLimitNode = new DistinctLimitNode( + Optional.empty(), + idAllocator.getNextId(), + keySource, + count, + false, + keys, + Optional.empty(), + SystemSessionProperties.getPrefilterForGroupbyLimitTimeoutMS(session)); + + FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager(); + RowExpression leftHashExpression = getHashExpression(functionAndTypeManager, keys).get(); + RowExpression rightHashExpression = getHashExpression(functionAndTypeManager, timedDistinctLimitNode.getOutputVariables()).get(); + + Type mapType = createMapType(functionAndTypeManager, BIGINT, BOOLEAN); + PlanNode rightProjectNode = projectExpressions(timedDistinctLimitNode, idAllocator, variableAllocator, ImmutableList.of(rightHashExpression, constant(TRUE, BOOLEAN)), ImmutableList.of()); + + VariableReferenceExpression mapAggVariable = variableAllocator.newVariable("expr", mapType); + PlanNode crossJoinRhs = addAggregation(rightProjectNode, functionAndTypeManager, idAllocator, variableAllocator, "MAP_AGG", mapType, ImmutableList.of(), mapAggVariable, rightProjectNode.getOutputVariables().get(0), rightProjectNode.getOutputVariables().get(1)); + PlanNode crossJoinLhs = addProjections(originalSource, idAllocator, variableAllocator, ImmutableList.of(leftHashExpression)); + ImmutableList.Builder crossJoinOutput = ImmutableList.builder(); + + crossJoinOutput.addAll(crossJoinLhs.getOutputVariables()); + crossJoinOutput.addAll(crossJoinRhs.getOutputVariables()); + + PlanNode crossJoin = new JoinNode( + Optional.empty(), + idAllocator.getNextId(), + typeConvert(Join.Type.CROSS), + crossJoinLhs, + crossJoinRhs, + ImmutableList.of(), + crossJoinOutput.build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(REPLICATED), + ImmutableMap.of()); + + VariableReferenceExpression mapVariable = crossJoinRhs.getOutputVariables().get(0); + VariableReferenceExpression lookupVariable = crossJoinLhs.getOutputVariables().get(crossJoinLhs.getOutputVariables().size() - 1); + RowExpression cardinality = call(functionAndTypeManager, "CARDINALITY", BIGINT, mapVariable); + RowExpression countExpr = constant(count, BIGINT); + + FunctionHandle equalsFunctionHandle = metadata.getFunctionAndTypeManager().resolveOperator(EQUAL, fromTypes(BIGINT, BIGINT)); + RowExpression foundAllEntires = call(EQUAL.name(), equalsFunctionHandle, BOOLEAN, cardinality, countExpr); + RowExpression mapElementAt = call(functionAndTypeManager, "element_at", BOOLEAN, mapVariable, lookupVariable); + RowExpression check = specialForm(IF, BOOLEAN, foundAllEntires, mapElementAt, constant(TRUE, BOOLEAN)); + + FilterNode filterNode = new FilterNode( + Optional.empty(), + idAllocator.getNextId(), + crossJoin, + check); + + Assignments.Builder originalOutputs = Assignments.builder(); + for (VariableReferenceExpression variableReferenceExpression : originalSource.getOutputVariables()) { + originalOutputs.put(variableReferenceExpression, variableReferenceExpression); + } + + ProjectNode filteredSource = new ProjectNode( + Optional.empty(), + idAllocator.getNextId(), + filterNode, + originalOutputs.build(), + LOCAL); + + return replaceChildren(aggregationNode, ImmutableList.of(filteredSource)); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index bf63612e31d4d..246b6ff87445d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -89,12 +89,25 @@ public class PushdownSubfields implements PlanOptimizer { private final Metadata metadata; + private boolean isEnabledForTesting; public PushdownSubfields(Metadata metadata) { this.metadata = requireNonNull(metadata, "metadata is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isPushdownSubfieldsEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { @@ -102,7 +115,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var requireNonNull(session, "session is null"); requireNonNull(types, "types is null"); - if (!isPushdownSubfieldsEnabled(session)) { + if (!isEnabled(session)) { return plan; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java index 464788b844fce..ed71aae2c84ad 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java @@ -142,16 +142,29 @@ public class RandomizeNullKeyInOuterJoin implements PlanOptimizer { private final FunctionAndTypeManager functionAndTypeManager; + private boolean isEnabledForTesting; public RandomizeNullKeyInOuterJoin(FunctionAndTypeManager functionAndTypeManager) { this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || getJoinDistributionType(session).canPartition() && !getRandomizeOuterJoinNullKeyStrategy(session).equals(DISABLED); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (getJoinDistributionType(session).canPartition() && !getRandomizeOuterJoinNullKeyStrategy(session).equals(DISABLED)) { + if (isEnabled(session)) { return SimplePlanRewriter.rewriteWith(new Rewriter(session, functionAndTypeManager, idAllocator, variableAllocator), plan, new HashSet<>()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RemoveRedundantDistinctAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RemoveRedundantDistinctAggregation.java index 7752792d49bc0..72dec36d419dd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RemoveRedundantDistinctAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RemoveRedundantDistinctAggregation.java @@ -54,10 +54,24 @@ public class RemoveRedundantDistinctAggregation implements PlanOptimizer { + private boolean isEnabledForTesting; + + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isRemoveRedundantDistinctAggregationEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (isRemoveRedundantDistinctAggregationEnabled(session)) { + if (isEnabled(session)) { PlanWithProperties result = new RemoveRedundantDistinctAggregation.Rewriter().accept(plan); return result.getNode(); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RewriteIfOverAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RewriteIfOverAggregation.java index 748cf38a2c855..78139b6958de0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RewriteIfOverAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RewriteIfOverAggregation.java @@ -87,12 +87,25 @@ public class RewriteIfOverAggregation implements PlanOptimizer { private final FunctionAndTypeManager functionAndTypeManager; + private boolean isEnabledForTesting; public RewriteIfOverAggregation(FunctionAndTypeManager functionAndTypeManager) { this.functionAndTypeManager = functionAndTypeManager; } + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isOptimizeConditionalAggregationEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, @@ -101,7 +114,7 @@ public PlanNode optimize(PlanNode plan, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (isOptimizeConditionalAggregationEnabled(session)) { + if (isEnabled(session)) { return SimplePlanRewriter.rewriteWith( new Rewriter(variableAllocator, idAllocator, new RowExpressionDeterminismEvaluator(functionAndTypeManager)), plan, ImmutableMap.of()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java index 92575fd04e6e7..bb6385afa3aff 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java @@ -102,10 +102,24 @@ public class SimplifyPlanWithEmptyInput implements PlanOptimizer { + private boolean isEnabledForTesting; + + @Override + public void setEnabledForTesting(boolean isSet) + { + isEnabledForTesting = isSet; + } + + @Override + public boolean isEnabled(Session session) + { + return isEnabledForTesting || isSimplifyPlanWithEmptyInputEnabled(session); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - if (isSimplifyPlanWithEmptyInputEnabled(session)) { + if (isEnabled(session)) { Rewriter rewriter = new Rewriter(idAllocator); PlanNode rewrittenNode = SimplePlanRewriter.rewriteWith(rewriter, plan); if (rewriter.isPlanChanged()) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index 95d8784960fc2..86b9d35eb0288 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -121,6 +121,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.execution.StageInfo.getAllStages; import static com.facebook.presto.expressions.DynamicFilters.extractDynamicFilters; @@ -179,7 +180,7 @@ private PlanPrinter( .mapToLong(planNode -> planNode.getPlanNodeScheduledTime().toMillis()) .sum(), MILLISECONDS)); - this.representation = new PlanRepresentation(planRoot, types, totalCpuTime, totalScheduledTime); + this.representation = new PlanRepresentation(planRoot, types, totalCpuTime, totalScheduledTime, session.getOptimizerInformationCollector().getOptimizationInfo()); RowExpressionFormatter rowExpressionFormatter = new RowExpressionFormatter(functionAndTypeManager); ConnectorSession connectorSession = requireNonNull(session, "session is null").toConnectorSession(); @@ -189,9 +190,9 @@ private PlanPrinter( planRoot.accept(visitor, null); } - public String toText(boolean verbose, int level) + public String toText(boolean verbose, int level, boolean verboseOptimizerInfo) { - return new TextRenderer(verbose, level).render(representation); + return new TextRenderer(verbose, level, verboseOptimizerInfo).render(representation); } public String toJson() @@ -208,7 +209,13 @@ public static String jsonFragmentPlan(PlanNode root, Set nodeInfo = new HashMap<>(); + private final List planOptimizerInfo; - public PlanRepresentation(PlanNode root, TypeProvider types, Optional totalCpuTime, Optional totalScheduledTime) + public PlanRepresentation(PlanNode root, TypeProvider types, Optional totalCpuTime, Optional totalScheduledTime, List planOptimizerInfo) { this.root = requireNonNull(root, "root is null"); this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null"); this.types = requireNonNull(types, "types is null"); this.totalScheduledTime = requireNonNull(totalScheduledTime, "totalScheduledTime is null"); + this.planOptimizerInfo = requireNonNull(planOptimizerInfo, "planOptimizerInfo is null"); } public NodeRepresentation getRoot() @@ -78,4 +82,9 @@ public void addNode(NodeRepresentation node) throw new IllegalStateException(String.format("Duplicate node ID %s: %s vs. %s", node.getId(), previous.getName(), node.getName())); } } + + public List getPlanOptimizerInfo() + { + return planOptimizerInfo; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java index 0877299ac5a67..326acd492e054 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java @@ -15,6 +15,7 @@ import com.facebook.presto.cost.PlanCostEstimate; import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.spi.eventlistener.PlanOptimizerInformation; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; @@ -38,18 +39,26 @@ public class TextRenderer { private final boolean verbose; private final int level; + private final boolean verboseOptimizerInfo; - public TextRenderer(boolean verbose, int level) + public TextRenderer(boolean verbose, int level, boolean verboseOptimizerInfo) { this.verbose = verbose; this.level = level; + this.verboseOptimizerInfo = verboseOptimizerInfo; } @Override public String render(PlanRepresentation plan) { StringBuilder output = new StringBuilder(); - return writeTextOutput(output, plan, level, plan.getRoot()); + String result = writeTextOutput(output, plan, level, plan.getRoot()); + + if (verboseOptimizerInfo) { + String optimizerInfo = optimizerInfoToText(plan.getPlanOptimizerInfo()); + result += optimizerInfo; + } + return result; } private String writeTextOutput(StringBuilder output, PlanRepresentation plan, int level, NodeRepresentation node) @@ -273,4 +282,20 @@ private static String indentMultilineString(String string, int level) { return string.replaceAll("(?m)^", indentString(level)); } + + private String optimizerInfoToText(List planOptimizerInfo) + { + List applicableOptimizers = planOptimizerInfo.stream() + .filter(x -> !x.getOptimizerTriggered() && x.getOptimizerApplicable().isPresent() && x.getOptimizerApplicable().get()) + .map(x -> x.getOptimizerName()).collect(toList()); + List triggeredOptimizers = planOptimizerInfo.stream() + .filter(x -> x.getOptimizerTriggered()) + .map(x -> x.getOptimizerName()).collect(toList()); + + String triggered = "Triggered optimizers: [" + + String.join(", ", triggeredOptimizers) + "]\n"; + String applicable = "Applicable optimizers: [" + + String.join(", ", applicableOptimizers) + "]\n"; + return triggered + applicable; + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java index 94f4629a39765..9fe9945ed5378 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java @@ -87,7 +87,8 @@ private PlanRepresentation getPlanRepresentation(PlanNode root) root, TypeProvider.viewOf(VARIABLE_ALLOCATOR.getVariables()), Optional.empty(), - Optional.empty()); + Optional.empty(), + ImmutableList.of()); } private NodeRepresentation getNodeRepresentation(PlanNode root, List planNodeIds) diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java new file mode 100644 index 0000000000000..a0d53f26ce515 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.tests; + +import com.facebook.presto.Session; +import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.testing.LocalQueryRunner; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tpch.TpchConnectorFactory; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_PAYLOAD_JOINS; +import static com.facebook.presto.testing.TestingSession.TESTING_CATALOG; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.testng.Assert.assertTrue; + +public class TestVerboseOptimizerInfo + extends AbstractTestQueries +{ + @Override + protected QueryRunner createQueryRunner() + { + return createLocalQueryRunner(); + } + + public static LocalQueryRunner createLocalQueryRunner() + { + Session defaultSession = testSessionBuilder() + .setCatalog("local") + .setSchema(TINY_SCHEMA_NAME) + .build(); + + LocalQueryRunner localQueryRunner = new LocalQueryRunner(defaultSession); + + // add the tpch catalog + // local queries run directly against the generator + localQueryRunner.createCatalog( + defaultSession.getCatalog().get(), + new TpchConnectorFactory(1), + ImmutableMap.of()); + + localQueryRunner.getMetadata().registerBuiltInFunctions(CUSTOM_FUNCTIONS); + + SessionPropertyManager sessionPropertyManager = localQueryRunner.getMetadata().getSessionPropertyManager(); + sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); + sessionPropertyManager.addConnectorSessionProperties(new ConnectorId(TESTING_CATALOG), TEST_CATALOG_PROPERTIES); + + return localQueryRunner; + } + + @Test + public void testApplicableOptimizers() + { + Session session = Session.builder(getSession()) + .setSystemProperty("verbose_optimizer_info_enabled", "true") + .build(); + String query = "SELECT o.orderkey FROM part p, orders o, lineitem l WHERE p.partkey = l.partkey AND l.orderkey = o.orderkey AND p.partkey <> o.orderkey AND p.name < l.comment"; + MaterializedResult materializedResult = computeActual(session, "explain " + query); + String explain = (String) getOnlyElement(materializedResult.getOnlyColumnAsSet()); + + checkOptimizerInfo(explain, true, ImmutableList.of("PruneCrossJoinColumns")); + checkOptimizerInfo(explain, false, ImmutableList.of("AddNotNullFiltersToJoinNode")); + + String payloadJoinQuery = "SELECT l.* FROM (select *, map(ARRAY[1,3], ARRAY[2,4]) as m1 from lineitem) l left join orders o on (l.orderkey = o.orderkey) left join part p on (l.partkey=p.partkey)"; + materializedResult = computeActual(session, "explain " + payloadJoinQuery); + String explainPayloadJoinQuery = (String) getOnlyElement(materializedResult.getOnlyColumnAsSet()); + + checkOptimizerInfo(explainPayloadJoinQuery, false, ImmutableList.of("PayloadJoinOptimizer")); + + Session sessionWithPayload = Session.builder(session) + .setSystemProperty(OPTIMIZE_PAYLOAD_JOINS, "true") + .build(); + materializedResult = computeActual(sessionWithPayload, "explain " + payloadJoinQuery); + explainPayloadJoinQuery = (String) getOnlyElement(materializedResult.getOnlyColumnAsSet()); + + checkOptimizerInfo(explainPayloadJoinQuery, true, ImmutableList.of("PayloadJoinOptimizer")); + } + + private void checkOptimizerInfo(String explain, boolean checkTriggered, List optimizers) + { + String regex = checkTriggered ? "Triggered optimizers.*" : "Applicable optimizers.*"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(explain); + assertTrue(matcher.find()); + + String optimizerInfo = matcher.group(); + for (String opt : optimizers) { + assertTrue(optimizerInfo.contains(opt)); + } + } +}