Merge pull request #602 from MarkMpn/ssms-21

Metadata improvements

.github/ISSUE_TEMPLATE/bug_report.md

@@ -29,3 +29,6 @@ If applicable, add screenshots to help explain your problem.
 
 **Additional context**
 Add any other context about the problem here.
+
+**Sponsorship**
+If you find this tool useful, please consider [sponsoring its development](https://github.com/sponsors/MarkMpn).

.github/ISSUE_TEMPLATE/feature_request.md

@@ -18,3 +18,6 @@ A clear and concise description of any alternative solutions or features you've
 
 **Additional context**
 Add any other context or screenshots about the feature request here.
+
+**Sponsorship**
+If you find this tool useful, please consider [sponsoring its development](https://github.com/sponsors/MarkMpn).

MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs

@@ -2930,7 +2930,7 @@ public void ErrorOnTruncateGuid()
         }
 
         [TestMethod]
-        public void ErrorOnTruncateEntityReferenc()
+        public void ErrorOnTruncateEntityReference()
         {
             using (var con = new Sql4CdsConnection(_localDataSources))
             using (var cmd = con.CreateCommand())
@@ -2951,5 +2951,33 @@ public void ErrorOnTruncateEntityReferenc()
                 }
             }
         }
+
+        [TestMethod]
+        public void NestedPrimaryFunctions()
+        {
+            using (var con = new Sql4CdsConnection(_localDataSources))
+            using (var cmd = con.CreateCommand())
+            {
+                cmd.CommandText = "INSERT INTO account (name) VALUES ('Data8')";
+                cmd.ExecuteNonQuery();
+                cmd.CommandText = "INSERT INTO account (name) VALUES (null)";
+                cmd.ExecuteNonQuery();
+
+                cmd.CommandText = "SELECT name, MAX(IIF(name = 'Data8', COALESCE(turnover, -1), 0)) FROM account GROUP BY name";
+
+                using (var reader = cmd.ExecuteReader())
+                {
+                    Assert.IsTrue(reader.Read());
+                    Assert.IsTrue(reader.IsDBNull(0));
+                    Assert.AreEqual(0, reader.GetDecimal(1));
+
+                    Assert.IsTrue(reader.Read());
+                    Assert.AreEqual("Data8", reader.GetString(0));
+                    Assert.AreEqual(-1, reader.GetDecimal(1));
+
+                    Assert.IsFalse(reader.Read());
+                }
+            }
+        }
     }
 }

MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs

@@ -8777,5 +8777,186 @@ metadata.entity e
             var fetch = AssertNode<FetchXmlScan>(loop.LeftSource);
             var meta = AssertNode<MetadataQueryNode>(loop.RightSource);
         }
+
+        class LoggingMetadataCache : IAttributeMetadataCache
+        {
+            private readonly IAttributeMetadataCache _cache;
+            private readonly Action<string> _log;
+
+            public LoggingMetadataCache(IAttributeMetadataCache cache, Action<string> log)
+            {
+                _cache = cache;
+                _log = log;
+            }
+
+            public EntityMetadata this[string name]
+            {
+                get
+                {
+                    _log(name);
+                    return _cache[name];
+                }
+            }
+
+            public EntityMetadata this[int otc]
+            {
+                get
+                {
+                    _log(otc.ToString());
+                    return _cache[otc];
+                }
+            }
+
+            public string[] RecycleBinEntities
+            {
+                get
+                {
+                    _log(nameof(RecycleBinEntities));
+                    return _cache.RecycleBinEntities;
+                }
+            }
+
+            public IEnumerable<EntityMetadata> GetAllEntities()
+            {
+                _log(nameof(GetAllEntities));
+                return _cache.GetAllEntities();
+            }
+
+            public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata)
+            {
+                _log($"{nameof(TryGetMinimalData)} {logicalName}");
+                return _cache.TryGetMinimalData(logicalName, out metadata);
+            }
+
+            public bool TryGetValue(string logicalName, out EntityMetadata metadata)
+            {
+                _log($"{nameof(TryGetValue)} {logicalName}");
+                return _cache.TryGetValue(logicalName, out metadata);
+            }
+        }
+
+        [TestMethod]
+        public void MinimalMetadata()
+        {
+            // https://github.com/MarkMpn/Sql4Cds/issues/593
+            var metadataLog = new HashSet<string>();
+            var dataSource = new FakeXrmDataSource
+            {
+                Name = "local",
+                Connection = _localDataSource.Connection,
+                Metadata = new LoggingMetadataCache(_localDataSource.Metadata, msg => metadataLog.Add(msg)),
+                TableSizeCache = _localDataSource.TableSizeCache,
+                MessageCache = _localDataSource.MessageCache
+            };
+            var planBuilder = new ExecutionPlanBuilder(new SessionContext(new Dictionary<string, DataSource> { [dataSource.Name] = dataSource }, this), this);
+
+            var query = "SELECT fullname FROM contact";
+
+            planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual("contact", String.Join(", ", metadataLog));
+        }
+
+        [TestMethod]
+        public void ColumnComparisonOnLinkEntity()
+        {
+            // https://github.com/MarkMpn/Sql4Cds/issues/595
+            // We can't use a column comparison condition inside a <link-entity> that references a sub-<link-entity>
+            // as filters on <link-entity> are used for join conditions to their parent and so the child table
+            // is not available at the point it is evaluated
+            var planBuilder = new ExecutionPlanBuilder(new SessionContext(_localDataSources, this), this);
+
+            var query = @"
+UPDATE c
+SET c.employees = au.ParentEmployees
+FROM account AS c INNER JOIN (
+  SELECT c1.accountid,
+         p.employees AS ParentEmployees
+  FROM   account c1 INNER JOIN account AS p ON c1.parentaccountid = p.accountid
+  WHERE  c1.employees <> p.employees
+  ) as au on c.parentaccountid = au.accountid";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual(1, plans.Length);
+
+            var update = AssertNode<UpdateNode>(plans[0]);
+            var join = AssertNode<MergeJoinNode>(update.Source);
+            var fetch1 = AssertNode<FetchXmlScan>(join.LeftSource);
+            Assert.AreEqual("au", fetch1.Alias);
+            AssertFetchXml(fetch1, @"
+<fetch>
+  <entity name='account'>
+    <attribute name='accountid' />
+    <link-entity name='account' to='parentaccountid' from='accountid' alias='p' link-type='inner'>
+      <attribute name='employees' alias='ParentEmployees' />
+      <filter>
+        <condition attribute='employees' operator='not-null' />
+      </filter>
+    </link-entity>
+    <filter>
+      <condition attribute='employees' operator='ne' valueof='p.employees' />
+      <condition attribute='employees' operator='not-null' />
+    </filter>
+    <order attribute='accountid' />
+  </entity>
+</fetch>");
+
+            var sort = AssertNode<SortNode>(join.RightSource);
+            var fetch2 = AssertNode<FetchXmlScan>(sort.Source);
+            Assert.AreEqual("c", fetch2.Alias);
+            AssertFetchXml(fetch2, @"
+<fetch>
+  <entity name='account'>
+    <attribute name='accountid' />
+    <attribute name='parentaccountid' />
+    <filter>
+      <condition attribute='parentaccountid' operator='not-null' />
+    </filter>
+  </entity>
+</fetch>");
+        }
+
+        [TestMethod]
+        public void TopClauseOnQueryDefinedTable()
+        {
+            var planBuilder = new ExecutionPlanBuilder(new SessionContext(_localDataSources, this), this);
+
+            var query = @"
+SELECT c.name, p.name
+FROM account AS c
+INNER JOIN (SELECT TOP 10 accountid, name FROM account) AS p ON c.parentaccountid = p.accountid";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual(1, plans.Length);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+            var join = AssertNode<MergeJoinNode>(select.Source);
+            var fetch1 = AssertNode<FetchXmlScan>(join.LeftSource);
+            Assert.AreEqual("p", fetch1.Alias);
+            AssertFetchXml(fetch1, @"
+<fetch top='10'>
+  <entity name='account'>
+    <attribute name='accountid' />
+    <attribute name='name' />
+    <order attribute='accountid' />
+  </entity>
+</fetch>");
+
+            var sort = AssertNode<SortNode>(join.RightSource);
+            var fetch2 = AssertNode<FetchXmlScan>(sort.Source);
+            Assert.AreEqual("c", fetch2.Alias);
+            AssertFetchXml(fetch2, @"
+<fetch>
+  <entity name='account'>
+    <attribute name='name' />
+    <attribute name='parentaccountid' />
+    <filter>
+      <condition attribute='parentaccountid' operator='not-null' />
+    </filter>
+  </entity>
+</fetch>");
+        }
     }
 }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs

@@ -250,7 +250,7 @@ private void AddSchemaColumn(string escapedAlias, string outputColumn, string so
                 return;
 
             var mapped = $"{escapedAlias}.{outputColumn}";
-            schema[mapped] = new ColumnDefinition(sourceSchema.Schema[normalized].Type, sourceSchema.Schema[normalized].IsNullable, false);
+            schema[mapped] = sourceSchema.Schema[normalized].NotCalculated();
             mappings[normalized] = mapped;
 
             if (normalized == sourceSchema.PrimaryKey)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs

@@ -182,7 +182,7 @@ protected virtual INodeSchema GetSchema(NodeCompilationContext context, bool inc
                         nullable = false;
                     }
 
-                    schema[column.Key] = new ColumnDefinition(column.Value.Type, nullable, column.Value.IsCalculated, column.Value.IsVisible);
+                    schema[column.Key] = nullable ? column.Value.Null() : column.Value.NotNull();
                 }
 
                 foreach (var alias in subSchema.Aliases)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/EntityReader.cs

@@ -396,7 +396,7 @@ private List<AttributeAccessor> ValidateInsertUpdateColumnMapping(DmlOperationDe
                         continue;
                     }
 
-                    targetType = virtualAttr.DataType;
+                    targetType = virtualAttr.DataType();
                     targetClrType = typeof(string);
 
                     if (!complexLookupAttributes.TryGetValue(attr.LogicalName, out var lookupDetails))

MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs

@@ -253,6 +253,11 @@ private void AddSchemaColumn(string name, DataTypeReference type)
             Schema[name] = new ColumnDefinition(type, true, false);
         }
 
+        private void AddSchemaColumn(string name, Func<DataTypeReference> typeLoader)
+        {
+            Schema[name] = new LazyColumnDefinition(typeLoader, true, false);
+        }
+
         private string PrefixWithAlias(string columnName)
         {
             if (String.IsNullOrEmpty(Alias))

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs

@@ -1446,7 +1446,7 @@ private void AddSchemaAttribute(DataSource dataSource, ColumnList schema, Dictio
             var notNull = innerJoin && attrMetadata.LogicalName == entityMetadata.PrimaryIdAttribute;
 
             // Add the logical attribute
-            AddSchemaAttribute(schema, aliases, fullName, simpleName, type, notNull);
+            AddSchemaAttribute(schema, aliases, fullName, simpleName, type, null, notNull);
 
             if (attrMetadata.IsPrimaryId == true)
                 _primaryKeyColumns[fullName] = attrMetadata.EntityLogicalName;
@@ -1456,10 +1456,10 @@ private void AddSchemaAttribute(DataSource dataSource, ColumnList schema, Dictio
 
             // Add standard virtual attributes
             foreach (var virtualAttr in attrMetadata.GetVirtualAttributes(dataSource, false))
-                AddSchemaAttribute(schema, aliases, AddSuffix(fullName, virtualAttr.Suffix), (attrMetadata.LogicalName + virtualAttr.Suffix).EscapeIdentifier(), virtualAttr.DataType, virtualAttr.NotNull ?? notNull);
+                AddSchemaAttribute(schema, aliases, AddSuffix(fullName, virtualAttr.Suffix), (attrMetadata.LogicalName + virtualAttr.Suffix).EscapeIdentifier(), null, virtualAttr.DataType, virtualAttr.NotNull ?? notNull);
         }
 
-        private void AddSchemaAttribute(ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, string fullName, string simpleName, DataTypeReference type, bool notNull)
+        private void AddSchemaAttribute(ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, string fullName, string simpleName, DataTypeReference type, Func<DataTypeReference> typeLoader, bool notNull)
         {
             var parts = fullName.SplitMultiPartIdentifier();
             var visible = true;
@@ -1471,7 +1471,10 @@ private void AddSchemaAttribute(ColumnList schema, Dictionary<string, IReadOnlyL
             if (_isVirtualEntity && type is SqlDataTypeReferenceWithCollation sqlType && sqlType.SqlDataTypeOption == SqlDataTypeOption.NVarChar)
                 type = DataTypeHelpers.NVarChar(Int32.MaxValue, sqlType.Collation, sqlType.CollationLabel);
 
-            schema[fullName] = new ColumnDefinition(type, !notNull, false, visible);
+            if (type != null)
+                schema[fullName] = new ColumnDefinition(type, !notNull, false, visible);
+            else
+                schema[fullName] = new LazyColumnDefinition(typeLoader, !notNull, false, visible);
 
             if (simpleName == null)
                 return;