Fold subqueries in join conditions to appropriate side of join

MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs

@@ -7553,10 +7553,10 @@ from account
 <fetch>
   <entity name='contact'>
     <all-attributes />
-    <link-entity name='new_customentity' to='firstname' from='new_name' link-type='in' />
     <link-entity name='account' to='parentcustomerid' from='accountid' alias='account' link-type='inner'>
       <all-attributes />
     </link-entity>
+    <link-entity name='new_customentity' to='firstname' from='new_name' link-type='in' />
   </entity>
 </fetch>");
             }
@@ -7586,15 +7586,135 @@ from account
 <fetch>
   <entity name='contact'>
     <all-attributes />
+    <link-entity name='account' to='parentcustomerid' from='accountid' alias='account' link-type='inner'>
+      <all-attributes />
+    </link-entity>
     <link-entity name='new_customentity' to='firstname' from='new_name' link-type='in' />
+  </entity>
+</fetch>");
+            }
+        }
+
+        [TestMethod]
+        public void SubqueryInJoinCriteriaLHS()
+        {
+            using (_localDataSource.EnableJoinOperator(JoinOperator.In))
+            {
+                var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this);
+
+                var query = @"
+select
+*
+from account
+inner join contact ON account.accountid = contact.parentcustomerid AND account.name IN (SELECT new_name FROM new_customentity)";
+
+                var plans = planBuilder.Build(query, null, out _);
+
+                Assert.AreEqual(1, plans.Length);
+
+                var select = AssertNode<SelectNode>(plans[0]);
+                var fetch = AssertNode<FetchXmlScan>(select.Source);
+
+                AssertFetchXml(fetch, @"
+<fetch>
+  <entity name='contact'>
+    <all-attributes />
     <link-entity name='account' to='parentcustomerid' from='accountid' alias='account' link-type='inner'>
       <all-attributes />
+      <link-entity name='new_customentity' to='name' from='new_name' link-type='in' />
     </link-entity>
   </entity>
 </fetch>");
             }
         }
 
+        [TestMethod]
+        public void SubqueryInJoinCriteriaLHSCorrelatedExists()
+        {
+            using (_localDataSource.EnableJoinOperator(JoinOperator.In))
+            {
+                var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this);
+
+                var query = @"
+select
+*
+from account
+inner join contact ON account.accountid = contact.parentcustomerid AND EXISTS(SELECT * FROM new_customentity WHERE new_name = account.name)";
+
+                var plans = planBuilder.Build(query, null, out _);
+
+                Assert.AreEqual(1, plans.Length);
+
+                var select = AssertNode<SelectNode>(plans[0]);
+                var fetch = AssertNode<FetchXmlScan>(select.Source);
+
+                AssertFetchXml(fetch, @"
+<fetch>
+  <entity name='contact'>
+    <all-attributes />
+    <link-entity name='account' to='parentcustomerid' from='accountid' alias='account' link-type='inner'>
+      <all-attributes />
+      <link-entity name='new_customentity' to='name' from='new_name' link-type='in' />
+    </link-entity>
+  </entity>
+</fetch>");
+            }
+        }
+
+        [TestMethod]
+        public void SubqueryInJoinCriteriaLHSAndRHSInnerJoin()
+        {
+            using (_localDataSource.EnableJoinOperator(JoinOperator.In))
+            {
+                var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this);
+
+                var query = @"
+select
+*
+from account
+inner join contact ON account.accountid = contact.parentcustomerid AND contact.fullname IN (SELECT new_name FROM new_customentity WHERE account.turnover = new_customentity.new_decimalprop)";
+
+                var plans = planBuilder.Build(query, null, out _);
+
+                Assert.AreEqual(1, plans.Length);
+
+                var select = AssertNode<SelectNode>(plans[0]);
+                var filter = AssertNode<FilterNode>(select.Source);
+                Assert.AreEqual("Expr2 IS NOT NULL", filter.Filter.ToSql());
+                var loop = AssertNode<NestedLoopNode>(filter.Source);
+                Assert.AreEqual(QualifiedJoinType.LeftOuter, loop.JoinType);
+                Assert.IsTrue(loop.SemiJoin);
+                Assert.AreEqual("contact.fullname = new_customentity.new_name", loop.JoinCondition.ToSql());
+                Assert.AreEqual(1, loop.OuterReferences.Count);
+                Assert.AreEqual("@Expr1", loop.OuterReferences["account.turnover"]);
+                Assert.AreEqual("new_customentity.new_name", loop.DefinedValues["Expr2"]);
+                var fetch1 = AssertNode<FetchXmlScan>(loop.LeftSource);
+                AssertFetchXml(fetch1, @"
+<fetch>
+  <entity name='contact'>
+    <all-attributes />
+    <link-entity name='account' to='parentcustomerid' from='accountid' alias='account' link-type='inner'>
+      <all-attributes />
+    </link-entity>
+  </entity>
+</fetch>");
+                var spool = AssertNode<IndexSpoolNode>(loop.RightSource);
+                Assert.AreEqual("new_customentity.new_decimalprop", spool.KeyColumn);
+                Assert.AreEqual("@Expr1", spool.SeekValue);
+                var fetch2 = AssertNode<FetchXmlScan>(spool.Source);
+                AssertFetchXml(fetch2, @"
+<fetch>
+  <entity name='new_customentity'>
+    <attribute name='new_name' />
+    <attribute name='new_decimalprop' />
+    <filter>
+      <condition attribute=""new_decimalprop"" operator=""not-null"" />
+    </filter>
+  </entity>
+</fetch>");
+            }
+        }
+
         [TestMethod]
         public void VirtualAttributeAliases()
         {

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs

@@ -165,6 +165,8 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
             foldedFilters |= ExpandFiltersOnColumnComparisons(context);
             foldedFilters |= FoldFiltersToDataSources(context, hints, subqueryConditions);
             foldedFilters |= FoldFiltersToInnerJoinSources(context, hints);
+            foldedFilters |= FoldFiltersToSpoolSource(context, hints);
+            foldedFilters |= FoldFiltersToNestedLoopCondition(context, hints);
 
             foreach (var addedLink in addedLinks)
             {
@@ -190,6 +192,60 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
             return this;
         }
 
+        private bool FoldFiltersToNestedLoopCondition(NodeCompilationContext context, IList<OptimizerHint> hints)
+        {
+            if (Filter == null)
+                return false;
+
+            if (!(Source is NestedLoopNode loop))
+                return false;
+
+            // Can't move the filter to the loop condition if we're using any of the defined values created by the loop
+            if (Filter.GetColumns().Any(c => loop.DefinedValues.ContainsKey(c)))
+                return false;
+
+            if (loop.JoinCondition == null)
+            {
+                loop.JoinCondition = Filter;
+            }
+            else
+            {
+                loop.JoinCondition = new BooleanBinaryExpression
+                {
+                    FirstExpression = loop.JoinCondition,
+                    BinaryExpressionType = BooleanBinaryExpressionType.And,
+                    SecondExpression = Filter
+                };
+            }
+
+            Filter = null;
+            return true;
+        }
+
+        private bool FoldFiltersToSpoolSource(NodeCompilationContext context, IList<OptimizerHint> hints)
+        {
+            if (Filter == null)
+                return false;
+
+            if (!(Source is TableSpoolNode spool))
+                return false;
+
+            var usesVariables = Filter.GetVariables().Any();
+
+            if (usesVariables)
+                return false;
+
+            spool.Source = new FilterNode
+            {
+                Source = spool.Source,
+                Filter = Filter
+            };
+
+            Filter = null;
+
+            return true;
+        }
+
         private bool FoldFiltersToInnerJoinSources(NodeCompilationContext context, IList<OptimizerHint> hints)
         {
             if (Filter == null)
@@ -219,7 +275,7 @@ private bool FoldFiltersToInnerJoinSources(NodeCompilationContext context, IList
 
             var rightContext = context;
 
-            if (join is NestedLoopNode loop)
+            if (join is NestedLoopNode loop && loop.OuterReferences != null)
             {
                 var innerParameterTypes = context.ParameterTypes
                     .Concat(loop.OuterReferences.Select(or => new KeyValuePair<string, DataTypeReference>(or.Value, leftSchema.Schema[or.Key].Type)))

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs

@@ -507,6 +507,9 @@ private bool FoldFetchXmlJoin(NodeCompilationContext context, IList<OptimizerHin
                         }
                     }
                 }
+
+                // Re-fold the FetchXML node to remove any filters that are now blank
+                leftFetch.FoldQuery(context, hints);
             }
 
             return true;