V9.3 fixes

MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs

@@ -2441,5 +2441,30 @@ SELECT CAST(CAST(10.6496 AS float) AS int) AS trunc1,
                 }
             }
         }
+
+        [TestMethod]
+        public void QueryDerivedTableWithContradiction()
+        {
+            // https://github.com/MarkMpn/Sql4Cds/issues/546
+            using (var con = new Sql4CdsConnection(_localDataSources))
+            using (var cmd = con.CreateCommand())
+            {
+                cmd.CommandTimeout = 0;
+
+                cmd.CommandText = @"
+SELECT *
+FROM (SELECT a.accountid
+      FROM   account AS a
+      WHERE  1 != 1) AS sub";
+
+                using (var reader = cmd.ExecuteReader())
+                {
+                    var schema = reader.GetSchemaTable();
+
+                    if (reader.Read())
+                        Assert.Fail();
+                }
+            }
+        }
     }
 }

MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs

@@ -8433,5 +8433,106 @@ FROM   metadata.alternate_key AS ak
             CollectionAssert.AreEqual(new[] { "EntityLogicalName", "LogicalName", "EntityKeyIndexStatus" }, meta.Query.KeyQuery.Properties.PropertyNames);
             Assert.AreEqual(0, meta.Query.KeyQuery.Criteria.Conditions.Count);
         }
+
+        [TestMethod]
+        public void OuterApplyOuterReference()
+        {
+            // https://github.com/MarkMpn/Sql4Cds/issues/547
+            var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this);
+
+            var query = @"
+SELECT *
+FROM (
+    SELECT a.accountid,
+           a.name
+    FROM   account a) AS q1
+FULL OUTER JOIN (
+    SELECT c.contactid,
+           c.parentcustomerid,
+           c.fullname
+    FROM   contact c) AS q2
+    ON q1.accountid = q2.parentcustomerid
+OUTER APPLY (
+    SELECT CASE WHEN q1.accountid = q2.parentcustomerid THEN 1 ELSE 0 END AS [flag]
+) AS q3
+WHERE q3.flag = 0";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual(1, plans.Length);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+            var apply = AssertNode<NestedLoopNode>(select.Source);
+            Assert.AreEqual(QualifiedJoinType.Inner, apply.JoinType);
+            var join = AssertNode<MergeJoinNode>(apply.LeftSource);
+            var fetch1 = AssertNode<FetchXmlScan>(join.LeftSource);
+            var sort = AssertNode<SortNode>(join.RightSource);
+            var fetch2 = AssertNode<FetchXmlScan>(sort.Source);
+            var alias = AssertNode<AliasNode>(apply.RightSource);
+            var filter = AssertNode<FilterNode>(alias.Source);
+            var compute = AssertNode<ComputeScalarNode>(filter.Source);
+            var constant = AssertNode<ConstantScanNode>(compute.Source);
+        }
+
+        [TestMethod]
+        public void FilterOnOuterApply()
+        {
+            // https://github.com/MarkMpn/Sql4Cds/issues/548
+            var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this);
+
+            var query = @"
+SELECT *
+FROM (
+    SELECT a.accountid,
+           a.name
+    FROM   account a) AS q1
+OUTER APPLY (
+    SELECT IIF(q1.name = 'Test1', 1, 0) AS [flag1],
+           IIF(q1.name = 'Test2', 1, 0) AS [flag2]
+) AS q2
+WHERE q2.flag1 = 1 OR q2.flag2 = 1";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual(1, plans.Length);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+            var filter = AssertNode<FilterNode>(select.Source);
+            Assert.AreEqual("q2.flag1 = 1 OR q2.flag2 = 1", filter.Filter.ToSql());
+            var apply = AssertNode<NestedLoopNode>(filter.Source);
+            Assert.AreEqual(QualifiedJoinType.LeftOuter, apply.JoinType);
+            Assert.IsNull(apply.JoinCondition);
+            var fetch = AssertNode<FetchXmlScan>(apply.LeftSource);
+            var alias = AssertNode<AliasNode>(apply.RightSource);
+            var compute = AssertNode<ComputeScalarNode>(alias.Source);
+            var constant = AssertNode<ConstantScanNode>(compute.Source);
+        }
+
+        [TestMethod]
+        public void ScalarSubqueryWithoutAlias()
+        {
+            var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this);
+
+            var query = @"
+SELECT a.accountid,
+(SELECT fullname FROM contact WHERE contactid = a.primarycontactid)
+FROM account a";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual(1, plans.Length);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+            var fetch = AssertNode<FetchXmlScan>(select.Source);
+            AssertFetchXml(fetch, @"
+<fetch>
+    <entity name='account'>
+        <attribute name='accountid' />
+        <link-entity name='contact' to='primarycontactid' from='contactid' alias='Expr2' link-type='outer'>
+            <attribute name='fullname' />
+        </link-entity>
+    </entity>
+</fetch>");
+        }
     }
 }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs

@@ -131,7 +131,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
             {
                 // Remove any unused columns
                 var unusedColumns = constant.Schema.Keys
-                    .Where(sourceCol => !ColumnSet.Any(col => col.SourceColumn.SplitMultiPartIdentifier().Last().EscapeIdentifier() == sourceCol))
+                    .Where(sourceCol => !ColumnSet.Any(col => (String.IsNullOrEmpty(constant.Alias) && col.SourceColumn == sourceCol) || (!String.IsNullOrEmpty(constant.Alias) && col.SourceColumn == constant.Alias.EscapeIdentifier() + "." + sourceCol)))
                     .ToList();
 
                 foreach (var col in unusedColumns)
@@ -145,7 +145,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
                 // Copy/rename any columns using the new aliases
                 foreach (var col in ColumnSet)
                 {
-                    var sourceColumn = col.SourceColumn.SplitMultiPartIdentifier().Last();
+                    var sourceColumn = constant.Alias == null ? col.SourceColumn : col.SourceColumn.SplitMultiPartIdentifier().Last();
 
                     if (String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != col.SourceColumn ||
                         !String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != constant.Alias.EscapeIdentifier() + "." + col.SourceColumn)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs

@@ -691,24 +691,44 @@ protected void ExecuteDmlOperation(DataSource dataSource, IQueryExecutionOptions
                                 else
                                     options.Progress(progress, $"{operationNames.InProgressUppercase} {newCount - threadCount + 1:N0}-{newCount:N0} of {entities.Count:N0} {GetDisplayName(0, meta)} ({progress:P0}, {threadCount:N0} threads)...");
 
-                                try
+                                while (true)
                                 {
-                                    var response = dataSource.Execute(threadLocalState.Service, request);
-                                    Interlocked.Increment(ref count);
-
-                                    responseHandler?.Invoke(response);
-                                }
-                                catch (FaultException<OrganizationServiceFault> ex)
-                                {
-                                    if (FilterErrors(context, request, ex.Detail))
+                                    try
                                     {
-                                        if (ContinueOnError)
-                                            fault = fault ?? ex.Detail;
-                                        else
-                                            throw;
+                                        var response = dataSource.Execute(threadLocalState.Service, request);
+                                        Interlocked.Increment(ref count);
+
+                                        responseHandler?.Invoke(response);
+                                        break;
                                     }
+                                    catch (FaultException<OrganizationServiceFault> ex)
+                                    {
+                                        if (ex.Detail.ErrorCode == 429 || // Virtual/elastic tables
+                                            ex.Detail.ErrorCode == -2147015902 || // Number of requests exceeded the limit of 6000 over time window of 300 seconds.
+                                            ex.Detail.ErrorCode == -2147015903 || // Combined execution time of incoming requests exceeded limit of 1,200,000 milliseconds over time window of 300 seconds. Decrease number of concurrent requests or reduce the duration of requests and try again later.
+                                            ex.Detail.ErrorCode == -2147015898) // Number of concurrent requests exceeded the limit of 52.
+                                        {
+                                            // In case throttling isn't handled by normal retry logic in the service client
+                                            var retryAfterSeconds = 2;
+
+                                            if (ex.Detail.ErrorDetails.TryGetValue("Retry-After", out var retryAfter) && (retryAfter is int || retryAfter is string s && Int32.TryParse(s, out _)))
+                                                retryAfterSeconds = Convert.ToInt32(retryAfter);
 
-                                    Interlocked.Increment(ref errorCount);
+                                            Thread.Sleep(retryAfterSeconds * 1000);
+                                            continue;
+                                        }
+
+                                        if (FilterErrors(context, request, ex.Detail))
+                                        {
+                                            if (ContinueOnError)
+                                                fault = fault ?? ex.Detail;
+                                            else
+                                                throw;
+                                        }
+
+                                        Interlocked.Increment(ref errorCount);
+                                        break;
+                                    }
                                 }
                             }
                             else

MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs

@@ -275,7 +275,9 @@ protected override bool FilterErrors(NodeExecutionContext context, OrganizationR
         {
             // Ignore errors trying to delete records that don't exist - record may have been deleted by another
             // process in parallel.
-            return fault.ErrorCode != -2147220891 && fault.ErrorCode != -2147185406 && fault.ErrorCode != -2147220969 && fault.ErrorCode != 404;
+            return fault.ErrorCode != -2147185406 && // IsvAbortedNotFound
+                fault.ErrorCode != -2147220969 && // ObjectDoesNotExist
+                fault.ErrorCode != 404; // Elastic tables
         }
 
         protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource, IOrganizationService org, EntityMetadata meta, ExecuteMultipleRequest req)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs

@@ -2144,6 +2144,9 @@ public void AddAliases(List<SelectColumn> columnSet, INodeSchema schema, IAttrib
                 .Select(g => g.Single())
                 .Where(c =>
                 {
+                    if (c.Alias == null)
+                        return false; // Don't fold null aliases, e.g. scalar subqueries
+
                     var parts = c.SourceColumn.SplitMultiPartIdentifier();
 
                     if (parts.Length > 1 && aliasStars.Contains(parts[0]))

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs

@@ -356,7 +356,7 @@ private bool FoldFiltersToNestedLoopCondition(NodeCompilationContext context, IL
             if (Filter == null)
                 return false;
 
-            if (!(Source is NestedLoopNode loop))
+            if (!(Source is NestedLoopNode loop) || loop.JoinType != QualifiedJoinType.Inner)
                 return false;
 
             // Can't move the filter to the loop condition if we're using any of the defined values created by the loop
@@ -790,7 +790,22 @@ private void ConvertOuterJoinsWithNonNullFiltersToInnerJoins(NodeCompilationCont
 
                 if (outerSource != null)
                 {
-                    var outerSchema = outerSource.GetSchema(context);
+                    // If we are enforcing a non-null constraint on the outer source, we can convert the join to an inner join
+                    // To get the schema of the outer source, we need to include any outer references that are used in the join condition
+                    var outerContext = context;
+
+                    if (join.JoinType == QualifiedJoinType.LeftOuter && join is NestedLoopNode loop && loop.OuterReferences != null && loop.OuterReferences.Count > 0)
+                    {
+                        var leftSchema = join.LeftSource.GetSchema(context);
+
+                        var innerParameterTypes = context.ParameterTypes
+                            .Concat(loop.OuterReferences.Select(or => new KeyValuePair<string, DataTypeReference>(or.Value, leftSchema.Schema[or.Key].Type)))
+                            .ToDictionary(p => p.Key, p => p.Value, StringComparer.OrdinalIgnoreCase);
+
+                        outerContext = new NodeCompilationContext(context, innerParameterTypes);
+                    }
+
+                    var outerSchema = outerSource.GetSchema(outerContext);
 
                     if (notNullColumns.Any(col => outerSchema.ContainsColumn(col, out _)))
                         join.JoinType = QualifiedJoinType.Inner;

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs

@@ -124,7 +124,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
 
             IDataExecutionPlanNodeInternal folded = null;
 
-            if (LeftAttributes.Count == 1 && ComparisonType == BooleanComparisonType.Equals)
+            if (LeftAttributes.Count == 1 && ComparisonType == BooleanComparisonType.Equals && JoinType != QualifiedJoinType.FullOuter)
             {
                 var leftFilter = JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.LeftOuter ? LeftSource as FilterNode : null;
                 var rightFilter = JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.RightOuter ? RightSource as FilterNode : null;

MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashJoinNode.cs

@@ -169,6 +169,8 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
                 LeftAttribute = RightAttribute;
                 RightAttribute = leftAttr;
 
+                (_leftKeyAccessors, _rightKeyAccessors) = (_rightKeyAccessors, _leftKeyAccessors);
+
                 if (JoinType == QualifiedJoinType.LeftOuter)
                     JoinType = QualifiedJoinType.RightOuter;
                 else if (JoinType == QualifiedJoinType.RightOuter)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs

@@ -313,7 +313,7 @@ protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource
             if (!req.Requests.All(r => r is CreateRequest))
                 return base.ExecuteMultiple(dataSource, org, meta, req);
 
-            if (meta.DataProviderId == DataProviders.ElasticDataProvider || dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "CreateMultiple"))
+            if (meta.DataProviderId == DataProviders.ElasticDataProvider || meta.DataProviderId == null && dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "CreateMultiple"))
             {
                 // Elastic tables can use CreateMultiple for better performance than ExecuteMultiple
                 var entities = new EntityCollection { EntityName = meta.LogicalName };

MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs

@@ -193,8 +193,9 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
                 var joinColumns = JoinCondition.GetColumns().ToList();
                 var hasLeftColumn = joinColumns.Any(c => leftSchema.ContainsColumn(c, out _));
                 var hasRightColumn = joinColumns.Any(c => rightSchema.ContainsColumn(c, out _));
+                var foldedFilter = false;
 
-                if (!hasLeftColumn)
+                if (!hasLeftColumn && JoinType == QualifiedJoinType.Inner)
                 {
                     // Join condition doesn't reference columns from the left source, so we can remove it from the join
                     // and apply it as a filter to the right source. Inner source will often have a table spool - add the filter
@@ -206,17 +207,19 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
 
                     RightSource = RightSource.FoldQuery(innerContext, hints);
                     RightSource.Parent = this;
+                    foldedFilter = true;
                 }
 
-                if (!hasRightColumn)
+                if (!hasRightColumn && (JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.LeftOuter))
                 {
                     // Join condition doesn't reference columns from the right source, so we can remove it from the join
                     // and apply it as a filter to the left source
                     LeftSource = new FilterNode { Source = LeftSource, Filter = JoinCondition }.FoldQuery(context, hints);
                     LeftSource.Parent = this;
+                    foldedFilter = true;
                 }
 
-                if (!hasLeftColumn || !hasRightColumn)
+                if (foldedFilter)
                     JoinCondition = null;
             }
 

MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs

@@ -591,15 +591,17 @@ protected override bool FilterErrors(NodeExecutionContext context, OrganizationR
         {
             // Ignore errors trying to update records that don't exist - record may have been deleted by another
             // process in parallel.
-            return fault.ErrorCode != -2147220969 && fault.ErrorCode != -2147185406 && fault.ErrorCode != -2147220969 && fault.ErrorCode != 404;
+            return fault.ErrorCode != -2147185406 && // IsvAbortedNotFound
+                fault.ErrorCode != -2147220969 && // ObjectDoesNotExist
+                fault.ErrorCode != 404; // Elastic tables
         }
 
         protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource, IOrganizationService org, EntityMetadata meta, ExecuteMultipleRequest req)
         {
             if (!req.Requests.All(r => r is UpdateRequest))
                 return base.ExecuteMultiple(dataSource, org, meta, req);
 
-            if (meta.DataProviderId == DataProviders.ElasticDataProvider || dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "UpdateMultiple"))
+            if (meta.DataProviderId == DataProviders.ElasticDataProvider || meta.DataProviderId == null && dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "UpdateMultiple"))
             {
                 // Elastic tables can use UpdateMultiple for better performance than ExecuteMultiple
                 var entities = new EntityCollection { EntityName = meta.LogicalName };