Virtual Entities

MarkMpn.Sql4Cds.Engine.Tests/MarkMpn.Sql4Cds.Engine.Tests.csproj

@@ -110,16 +110,6 @@
     <None Include="app.config" />
     <None Include="Key.snk" />
   </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\MarkMpn.Sql4Cds.Engine\MarkMpn.Sql4Cds.Engine.csproj">
-      <Project>{c77b731d-e55c-4197-b96c-2b23eb9f56ef}</Project>
-      <Name>MarkMpn.Sql4Cds.Engine</Name>
-    </ProjectReference>
-    <ProjectReference Include="..\MarkMpn.Sql4Cds\MarkMpn.Sql4Cds.csproj">
-      <Project>{a7af5d13-a44e-426d-b3fc-ae390832c7df}</Project>
-      <Name>MarkMpn.Sql4Cds</Name>
-    </ProjectReference>
-  </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Dapper.StrongName">
       <Version>2.1.28</Version>
@@ -140,6 +130,12 @@
       <Version>6.0.7</Version>
     </PackageReference>
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\MarkMpn.Sql4Cds.Engine\MarkMpn.Sql4Cds.Engine.csproj">
+      <Project>{c77b731d-e55c-4197-b96c-2b23eb9f56ef}</Project>
+      <Name>MarkMpn.Sql4Cds.Engine</Name>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
 </Project>

MarkMpn.Sql4Cds.Engine/ExecutionPlan/DistinctNode.cs

@@ -119,7 +119,9 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
                     }
                 }
 
-                if (!virtualAttr)
+                // Virtual entity providers are unreliable - still fold the DISTINCT to the fetch but keep
+                // this node to ensure the DISTINCT is applied if the provider doesn't support it.
+                if (!virtualAttr && !fetch.IsUnreliableVirtualEntityProvider)
                     return fetch;
 
                 schema = Source.GetSchema(context);

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs

@@ -91,6 +91,7 @@ public InvalidPagingException(string message) : base(message)
         private List<KeyValuePair<string, string>> _pagingFields;
         private List<INullable> _lastPageValues;
         private bool _missingPagingCookie;
+        private bool _isVirtualEntity;
 
         public FetchXmlScan()
         {
@@ -195,6 +196,8 @@ public FetchXmlScan()
         [Browsable(false)]
         public string Warning => _missingPagingCookie && RowsOut == 50_000 ? "Using legacy paging - results may be incomplete" : null;
 
+        internal bool IsUnreliableVirtualEntityProvider => _isVirtualEntity;
+
         public bool RequiresCustomPaging(IDictionary<string, DataSource> dataSources)
         {
             // Never need to do paging if we're enforcing a TOP constraint
@@ -291,6 +294,7 @@ protected override IEnumerable<Entity> ExecuteInternal(NodeExecutionContext cont
             var mainEntity = FetchXml.Items.OfType<FetchEntityType>().Single();
             var name = mainEntity.name;
             var meta = dataSource.Metadata[name];
+            _isVirtualEntity = meta.DataProviderId != null && meta.DataProviderId != DataProviders.ElasticDataProvider;
 
             if (!(Parent is PartitionedAggregateNode))
                 context.Options.Progress(0, $"Retrieving {GetDisplayName(0, meta)}...");
@@ -748,10 +752,69 @@ private void OnRetrievedEntity(Entity entity, INodeSchema schema, IQueryExecutio
             {
                 object sqlValue;
 
-                if (entity.Attributes.TryGetValue(col.Key, out var value) && value != null)
-                    sqlValue = SqlTypeConverter.NetToSqlType(dataSource, value, col.Value.Type);
-                else
+                if (!entity.Attributes.TryGetValue(col.Key, out var value) && _isVirtualEntity)
+                {
+                    // Virtual entity providers aren't reliable and can produce attributes with names in different cases
+                    // than expected, e.g. msdyn_solutioncomponentsummary returns its id as msdyn_solutioncomponentsummaryId
+                    var altKey = entity.Attributes.Keys.FirstOrDefault(k => k.Equals(col.Key, StringComparison.OrdinalIgnoreCase));
+
+                    if (altKey != null)
+                    {
+                        value = entity[col.Key] = entity[altKey];
+                        entity.Attributes.Remove(altKey);
+                    }
+                }
+
+                if (value == null)
+                {
                     sqlValue = SqlTypeConverter.GetNullValue(col.Value.Type.ToNetType(out _));
+                }
+                else
+                {
+                    if (_isVirtualEntity)
+                    {
+                        // Virtual entity providers aren't reliable and can produce attribute values of different types
+                        // than expected, e.g. msdyn_componentlayer returns string values as guids. Convert the CLR
+                        // values to the correct type before converting to SQL types.
+                        var expectedClrType = SqlTypeConverter.SqlToNetType(col.Value.Type.ToNetType(out _));
+                        if (value.GetType() != expectedClrType)
+                        {
+                            if (value is Guid guidValue)
+                            {
+                                if (expectedClrType == typeof(string))
+                                {
+                                    value = guidValue.ToString();
+                                }
+                                else if (expectedClrType == typeof(EntityReference))
+                                {
+                                    // We don't know the logical name of the entity reference, check if we can find it from the metadata
+                                    var parts = col.Key.SplitMultiPartIdentifier();
+                                    var entityLogicalName = Entity.name;
+                                    if (!parts[0].Equals(Alias, StringComparison.OrdinalIgnoreCase))
+                                        entityLogicalName = Entity.FindLinkEntity(parts[0]).name;
+                                    var attrMeta = dataSource.Metadata[entityLogicalName].Attributes.Single(a => a.LogicalName == parts[1]);
+                                    if (attrMeta.IsPrimaryId == false && attrMeta is LookupAttributeMetadata lookupAttrMeta)
+                                    {
+                                        if (lookupAttrMeta.Targets?.Length == 1)
+                                            value = new EntityReference(lookupAttrMeta.Targets[0], guidValue);
+                                        else
+                                            value = new EntityReference(null, guidValue);
+                                    }
+                                }
+                                else
+                                {
+                                    throw new QueryExecutionException($"Expected {expectedClrType.Name} value, got {value.GetType()}");
+                                }
+                            }
+                            else
+                            {
+                                value = Convert.ChangeType(value, expectedClrType);
+                            }
+                        }
+                    }
+
+                    sqlValue = SqlTypeConverter.NetToSqlType(dataSource, value, col.Value.Type);
+                }
 
                 if (_primaryKeyColumns.TryGetValue(col.Key, out var logicalName) && sqlValue is SqlGuid guid)
                     sqlValue = new SqlEntityReference(DataSource, logicalName, guid);
@@ -955,6 +1018,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
             // Add each attribute from the main entity and recurse into link entities
             var entity = FetchXml.Items.OfType<FetchEntityType>().Single();
             var meta = dataSource.Metadata[entity.name];
+            _isVirtualEntity = meta.DataProviderId != null && meta.DataProviderId != DataProviders.ElasticDataProvider;
 
             var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
@@ -1333,6 +1397,11 @@ private void AddSchemaAttribute(ColumnList schema, Dictionary<string, IReadOnlyL
             if (parts.Length == 2 && HiddenAliases.Contains(parts[0]))
                 visible = false;
 
+            // Virtual entity providers are not reliable - they can produce string values that are longer than the metadata indicates
+            // e.g. msdn_componentlayer.msdyn_children
+            if (_isVirtualEntity && type is SqlDataTypeReferenceWithCollation sqlType && sqlType.SqlDataTypeOption == SqlDataTypeOption.NVarChar)
+                type = DataTypeHelpers.NVarChar(Int32.MaxValue, sqlType.Collation, sqlType.CollationLabel);
+
             schema[fullName] = new ColumnDefinition(type, !notNull, false, visible);
 
             if (simpleName == null)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/OffsetFetchNode.cs

@@ -73,6 +73,10 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
 
             if (Source is FetchXmlScan fetchXml)
             {
+                // Virtual entity providers aren't reliable - use naive implementation
+                if (fetchXml.IsUnreliableVirtualEntityProvider)
+                    return this;
+
                 var expressionExecutionContext = new ExpressionExecutionContext(expressionCompilationContext);
                 var offset = SqlTypeConverter.ChangeType<int>(offsetLiteral.Compile(expressionCompilationContext)(expressionExecutionContext));
                 var count = SqlTypeConverter.ChangeType<int>(fetchLiteral.Compile(expressionCompilationContext)(expressionExecutionContext));

MarkMpn.Sql4Cds.Engine/ExecutionPlan/SortNode.cs

@@ -519,6 +519,14 @@ private IDataExecutionPlanNodeInternal FoldSorts(NodeCompilationContext context)
                         fetchXml.FetchXml.UseRawOrderBy = true;
                 }
 
+                // Virtual entity providers are unreliable - fold the sorts to the FetchXML but keep this
+                // node to resort if required.
+                if (fetchXml.IsUnreliableVirtualEntityProvider)
+                {
+                    PresortedCount = 0;
+                    return this;
+                }
+
                 return Source;
             }
 

MarkMpn.Sql4Cds.Engine/ExecutionPlan/TopNode.cs

@@ -169,6 +169,11 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
                         fetchXml.FetchXml.top = literal.Value;
                         fetchXml.AllPages = false;
 
+                        // Virtual entity providers aren't reliable - fold the TOP into the FetchXML but keep
+                        // this node in case the provider doesn't support TOP
+                        if (fetchXml.IsUnreliableVirtualEntityProvider)
+                            return this;
+
                         if (Source == fetchXml)
                             return fetchXml;