Track applicable optimizations that were disabled with a flag

presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java

@@ -237,6 +237,7 @@ public final class SystemSessionProperties
     public static final String EXCEEDED_MEMORY_LIMIT_HEAP_DUMP_FILE_DIRECTORY = "exceeded_memory_limit_heap_dump_file_directory";
     public static final String DISTRIBUTED_TRACING_MODE = "distributed_tracing_mode";
     public static final String VERBOSE_RUNTIME_STATS_ENABLED = "verbose_runtime_stats_enabled";
+    public static final String VERBOSE_OPTIMIZER_INFO_ENABLED = "verbose_optimizer_info_enabled";
     public static final String STREAMING_FOR_PARTIAL_AGGREGATION_ENABLED = "streaming_for_partial_aggregation_enabled";
     public static final String MAX_STAGE_COUNT_FOR_EAGER_SCHEDULING = "max_stage_count_for_eager_scheduling";
     public static final String HYPERLOGLOG_STANDARD_ERROR_WARNING_THRESHOLD = "hyperloglog_standard_error_warning_threshold";
@@ -1292,6 +1293,11 @@ public SystemSessionProperties(
                         "Enable logging all runtime stats",
                         featuresConfig.isVerboseRuntimeStatsEnabled(),
                         false),
+                booleanProperty(
+                        VERBOSE_OPTIMIZER_INFO_ENABLED,
+                        "Enable logging of verbose information about applied optimizations",
+                        featuresConfig.isVerboseOptimizerInfoEnabled(),
+                        false),
                 booleanProperty(
                         STREAMING_FOR_PARTIAL_AGGREGATION_ENABLED,
                         "Enable streaming for partial aggregation",
@@ -2460,6 +2466,11 @@ public static boolean isVerboseRuntimeStatsEnabled(Session session)
         return session.getSystemProperty(VERBOSE_RUNTIME_STATS_ENABLED, Boolean.class);
     }
 
+    public static boolean isVerboseOptimizerInfoEnabled(Session session)
+    {
+        return session.getSystemProperty(VERBOSE_OPTIMIZER_INFO_ENABLED, Boolean.class);
+    }
+
     public static boolean isLeafNodeLimitEnabled(Session session)
     {
         return session.getSystemProperty(LEAF_NODE_LIMIT_ENABLED, Boolean.class);

presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java

@@ -26,11 +26,13 @@
 import com.facebook.presto.spi.PrestoException;
 import com.facebook.presto.spi.VariableAllocator;
 import com.facebook.presto.spi.WarningCollector;
+import com.facebook.presto.spi.eventlistener.PlanOptimizerInformation;
 import com.facebook.presto.spi.plan.PlanNode;
 import com.facebook.presto.spi.plan.PlanNodeIdAllocator;
 import com.facebook.presto.sql.parser.SqlParser;
 import com.facebook.presto.sql.planner.Plan;
 import com.facebook.presto.sql.planner.TypeProvider;
+import com.facebook.presto.sql.planner.iterative.IterativeOptimizer;
 import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher;
 import com.facebook.presto.sql.planner.optimizations.PlanOptimizer;
 import com.facebook.presto.sql.planner.plan.JoinNode;
@@ -42,6 +44,7 @@
 
 import static com.facebook.presto.SystemSessionProperties.getQueryAnalyzerTimeout;
 import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery;
+import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled;
 import static com.facebook.presto.common.RuntimeUnit.NANO;
 import static com.facebook.presto.spi.StandardErrorCode.QUERY_PLANNING_TIMEOUT;
 import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED;
@@ -105,11 +108,13 @@ public Plan validateAndOptimizePlan(PlanNode root, PlanStage stage)
                     throw new PrestoException(QUERY_PLANNING_TIMEOUT, String.format("The query optimizer exceeded the timeout of %s.", getQueryAnalyzerTimeout(session).toString()));
                 }
                 long start = System.nanoTime();
-                root = optimizer.optimize(root, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector);
-                requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName()));
+                PlanNode newRoot = optimizer.optimize(root, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector);
+                requireNonNull(newRoot, format("%s returned a null plan", optimizer.getClass().getName()));
                 if (enableVerboseRuntimeStats) {
                     session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", optimizer.getClass().getSimpleName()), NANO, System.nanoTime() - start);
                 }
+                collectOptimizerInformation(optimizer, root, newRoot);
+                root = newRoot;
             }
         }
 
@@ -133,4 +138,22 @@ private StatsAndCosts computeStats(PlanNode root, TypeProvider types)
         }
         return StatsAndCosts.empty();
     }
+
+    private void collectOptimizerInformation(PlanOptimizer optimizer, PlanNode oldNode, PlanNode newNode)
+    {
+        if (optimizer instanceof IterativeOptimizer) {
+            // iterative optimizers do their own recording of what rules got triggered
+            return;
+        }
+
+        boolean isTriggered = (oldNode != newNode);
+        boolean isApplicable =
+                isTriggered ||
+                !optimizer.isEnabled(session) && isVerboseOptimizerInfoEnabled(session) &&
+                        optimizer.isApplicable(oldNode, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector);
+
+        if (isTriggered || isApplicable) {
+            session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(optimizer.getClass().getSimpleName(), isTriggered, Optional.of(isApplicable)));
+        }
+    }
 }

presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java

@@ -234,6 +234,7 @@ public class FeaturesConfig
     private AggregationIfToFilterRewriteStrategy aggregationIfToFilterRewriteStrategy = AggregationIfToFilterRewriteStrategy.DISABLED;
     private String analyzerType = "BUILTIN";
     private boolean verboseRuntimeStatsEnabled;
+    private boolean verboseOptimizerInfoEnabled;
 
     private boolean streamingForPartialAggregationEnabled;
     private boolean preferMergeJoinForSortedInputs;
@@ -2205,6 +2206,11 @@ public boolean isVerboseRuntimeStatsEnabled()
         return verboseRuntimeStatsEnabled;
     }
 
+    public boolean isVerboseOptimizerInfoEnabled()
+    {
+        return verboseOptimizerInfoEnabled;
+    }
+
     @Config("verbose-runtime-stats-enabled")
     @ConfigDescription("Enable logging all runtime stats.")
     public FeaturesConfig setVerboseRuntimeStatsEnabled(boolean value)

presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java

@@ -43,6 +43,7 @@
 import java.util.Set;
 import java.util.stream.Stream;
 
+import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled;
 import static com.facebook.presto.common.RuntimeUnit.NANO;
 import static com.facebook.presto.spi.StandardErrorCode.OPTIMIZER_TIMEOUT;
 import static com.google.common.base.Preconditions.checkArgument;
@@ -167,6 +168,9 @@ private boolean exploreNode(int group, Context context, Matcher matcher)
                 Rule<?> rule = possiblyMatchingRules.next();
 
                 if (!rule.isEnabled(context.session)) {
+                    if (isVerboseOptimizerInfoEnabled(context.session) && isApplicable(node, rule, matcher, context)) {
+                        context.addRulesApplicable(rule.getClass().getSimpleName());
+                    }
                     continue;
                 }
 
@@ -224,6 +228,17 @@ private <T> Rule.Result transform(PlanNode node, Rule<T> rule, Matcher matcher,
         return result;
     }
 
+    private <T> boolean isApplicable(PlanNode node, Rule<T> rule, Matcher matcher, Context context)
+    {
+        Match<T> match = matcher.match(rule.getPattern(), node);
+        if (match.isEmpty()) {
+            return false;
+        }
+
+        Rule.Result result = rule.apply(match.value(), match.captures(), ruleContext(context));
+        return !result.isEmpty();
+    }
+
     private boolean exploreChildren(int group, Context context, Matcher matcher)
     {
         boolean progress = false;
@@ -313,6 +328,7 @@ private static class Context
         private final CostProvider costProvider;
         private final StatsProvider statsProvider;
         private final Set<String> rulesTriggered;
+        private final Set<String> rulesApplicable;
 
         public Context(
                 Memo memo,
@@ -339,6 +355,7 @@ public Context(
             this.costProvider = costProvider;
             this.statsProvider = statsProvider;
             this.rulesTriggered = new HashSet<>();
+            this.rulesApplicable = new HashSet<>();
         }
 
         public void checkTimeoutNotExhausted()
@@ -353,9 +370,15 @@ public void addRulesTriggered(String rule)
             rulesTriggered.add(rule);
         }
 
+        public void addRulesApplicable(String rule)
+        {
+            rulesApplicable.add(rule);
+        }
+
         public void collectOptimizerInformation()
         {
             rulesTriggered.forEach(x -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(x, true, Optional.empty())));
+            rulesApplicable.forEach(x -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(x, false, Optional.of(true))));
         }
     }
 }

presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java

@@ -93,12 +93,25 @@ public class HashGenerationOptimizer
         implements PlanOptimizer
 {
     private final FunctionAndTypeManager functionAndTypeManager;
+    private boolean isEnabledForTesting;
 
     public HashGenerationOptimizer(FunctionAndTypeManager functionAndTypeManager)
     {
         this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionManager is null");
     }
 
+    @Override
+    public void setEnabledForTesting(boolean isSet)
+    {
+        isEnabledForTesting = isSet;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isEnabledForTesting || SystemSessionProperties.isOptimizeHashGenerationEnabled(session);
+    }
+
     @Override
     public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector)
     {
@@ -107,7 +120,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var
         requireNonNull(types, "types is null");
         requireNonNull(variableAllocator, "variableAllocator is null");
         requireNonNull(idAllocator, "idAllocator is null");
-        if (SystemSessionProperties.isOptimizeHashGenerationEnabled(session)) {
+        if (isEnabled(session)) {
             PlanWithProperties result = new Rewriter(idAllocator, variableAllocator, functionAndTypeManager).accept(plan, new HashComputationSet());
             return result.getNode();
         }

presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java

@@ -44,12 +44,25 @@ public class HistoricalStatisticsEquivalentPlanMarkingOptimizer
     private static final Set<Class<? extends PlanNode>> LIMITING_NODES =
             ImmutableSet.of(TopNNode.class, LimitNode.class, DistinctLimitNode.class, TopNRowNumberNode.class);
     private final StatsCalculator statsCalculator;
+    private boolean isEnabledForTesting;
 
     public HistoricalStatisticsEquivalentPlanMarkingOptimizer(StatsCalculator statsCalculator)
     {
         this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null");
     }
 
+    @Override
+    public void setEnabledForTesting(boolean isSet)
+    {
+        isEnabledForTesting = isSet;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isEnabledForTesting || useHistoryBasedPlanStatisticsEnabled(session) || trackHistoryBasedPlanStatisticsEnabled(session);
+    }
+
     @Override
     public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector)
     {
@@ -59,7 +72,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var
         requireNonNull(variableAllocator, "variableAllocator is null");
         requireNonNull(idAllocator, "idAllocator is null");
 
-        if (!useHistoryBasedPlanStatisticsEnabled(session) && !trackHistoryBasedPlanStatisticsEnabled(session)) {
+        if (!isEnabled(session)) {
             return plan;
         }
 

presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java

@@ -68,23 +68,39 @@ public class KeyBasedSampler
         implements PlanOptimizer
 {
     private final Metadata metadata;
+    private boolean isEnabledForTesting;
 
     public KeyBasedSampler(Metadata metadata, SqlParser sqlParser)
     {
         this.metadata = requireNonNull(metadata, "metadata is null");
     }
 
+    @Override
+    public void setEnabledForTesting(boolean isSet)
+    {
+        isEnabledForTesting = isSet;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isEnabledForTesting || isKeyBasedSamplingEnabled(session);
+    }
+
     @Override
     public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector)
     {
-        if (isKeyBasedSamplingEnabled(session)) {
+        if (isEnabled(session)) {
             List<String> sampledFields = new ArrayList<>(2);
             PlanNode rewritten = SimplePlanRewriter.rewriteWith(new Rewriter(session, metadata.getFunctionAndTypeManager(), idAllocator, sampledFields), plan, null);
-            if (!sampledFields.isEmpty()) {
-                warningCollector.add(new PrestoWarning(SAMPLED_FIELDS, String.format("Sampled the following columns/derived columns at %s percent:%n\t%s", getKeyBasedSamplingPercentage(session) * 100., String.join("\n\t", sampledFields))));
-            }
-            else {
-                warningCollector.add(new PrestoWarning(SEMANTIC_WARNING, "Sampling could not be performed due to the query structure"));
+
+            if (!isEnabledForTesting) {
+                if (!sampledFields.isEmpty()) {
+                    warningCollector.add(new PrestoWarning(SAMPLED_FIELDS, String.format("Sampled the following columns/derived columns at %s percent:%n\t%s", getKeyBasedSamplingPercentage(session) * 100., String.join("\n\t", sampledFields))));
+                }
+                else {
+                    warningCollector.add(new PrestoWarning(SEMANTIC_WARNING, "Sampling could not be performed due to the query structure"));
+                }
             }
 
             return rewritten;

presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java

@@ -40,13 +40,26 @@ public class MergeJoinForSortedInputOptimizer
 {
     private final Metadata metadata;
     private final SqlParser parser;
+    private boolean isEnabledForTesting;
 
     public MergeJoinForSortedInputOptimizer(Metadata metadata, SqlParser parser)
     {
         this.metadata = requireNonNull(metadata, "metadata is null");
         this.parser = requireNonNull(parser, "parser is null");
     }
 
+    @Override
+    public void setEnabledForTesting(boolean isSet)
+    {
+        isEnabledForTesting = isSet;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isEnabledForTesting || isGroupedExecutionEnabled(session) && preferMergeJoinForSortedInputs(session);
+    }
+
     @Override
     public PlanNode optimize(PlanNode plan, Session session, TypeProvider type, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector)
     {
@@ -55,7 +68,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider type, Vari
         requireNonNull(variableAllocator, "variableAllocator is null");
         requireNonNull(idAllocator, "idAllocator is null");
 
-        if (isGroupedExecutionEnabled(session) && preferMergeJoinForSortedInputs(session)) {
+        if (isEnabled(session)) {
             return SimplePlanRewriter.rewriteWith(new MergeJoinForSortedInputOptimizer.Rewriter(variableAllocator, idAllocator, metadata, session), plan, null);
         }
         return plan;

presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergePartialAggregationsWithFilter.java

@@ -90,16 +90,29 @@ public class MergePartialAggregationsWithFilter
         implements PlanOptimizer
 {
     private final FunctionAndTypeManager functionAndTypeManager;
+    private boolean isEnabledForTesting;
 
     public MergePartialAggregationsWithFilter(FunctionAndTypeManager functionAndTypeManager)
     {
         this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null");
     }
 
+    @Override
+    public void setEnabledForTesting(boolean isSet)
+    {
+        isEnabledForTesting = isSet;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isEnabledForTesting || isMergeAggregationsWithAndWithoutFilter(session);
+    }
+
     @Override
     public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector)
     {
-        if (isMergeAggregationsWithAndWithoutFilter(session)) {
+        if (isEnabled(session)) {
             return SimplePlanRewriter.rewriteWith(new Rewriter(session, variableAllocator, idAllocator, functionAndTypeManager), plan, new Context());
         }
 

presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java

@@ -83,17 +83,30 @@ public class OptimizeMixedDistinctAggregations
 {
     private final Metadata metadata;
     private final StandardFunctionResolution functionResolution;
+    private boolean isEnabledForTesting;
 
     public OptimizeMixedDistinctAggregations(Metadata metadata)
     {
         this.metadata = metadata;
         this.functionResolution = new FunctionResolution(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver());
     }
 
+    @Override
+    public void setEnabledForTesting(boolean isSet)
+    {
+        isEnabledForTesting = isSet;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isEnabledForTesting || isOptimizeDistinctAggregationEnabled(session);
+    }
+
     @Override
     public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector)
     {
-        if (isOptimizeDistinctAggregationEnabled(session)) {
+        if (isEnabled(session)) {
             return SimplePlanRewriter.rewriteWith(new Optimizer(idAllocator, variableAllocator, metadata, functionResolution), plan, Optional.empty());
         }