Merge branch 'main' into lucgen/issue2780

src/libraries/Microsoft.PowerFx.Connectors/Tabular/Capabilities/CapabilityConstants.cs

@@ -36,6 +36,7 @@ internal static class CapabilityConstants
         public const string SPIsChoice = "IsChoice";
         public const string SPQueryName = "OdataQueryName";
         public const string SupportsRecordPermission = "supportsRecordPermission";
+        public const string SupportsJoin = "supportsJoin";
         public const string UngroupableProperties = "ungroupableProperties";
         public const string UnsortableProperties = "unsortableProperties";        
     }

src/libraries/Microsoft.PowerFx.Connectors/Tabular/Capabilities/ServiceCapabilities.cs

@@ -94,10 +94,15 @@ internal sealed class ServiceCapabilities : IColumnsCapabilities
         [JsonPropertyName(CapabilityConstants.SupportsRecordPermission)]
         public readonly bool SupportsRecordPermission;
 
+        [JsonInclude]
+        [JsonPropertyName(CapabilityConstants.SupportsJoin)]
+        public readonly bool SupportsJoinFunction;
+
         public const int CurrentODataVersion = 4;
 
         public ServiceCapabilities(SortRestriction sortRestriction, FilterRestriction filterRestriction, SelectionRestriction selectionRestriction, GroupRestriction groupRestriction, IEnumerable<string> filterFunctions,
-                                   IEnumerable<string> filterSupportedFunctions, PagingCapabilities pagingCapabilities, bool recordPermissionCapabilities, int oDataVersion = CurrentODataVersion, bool supportsDataverseOffline = false)
+                                   IEnumerable<string> filterSupportedFunctions, PagingCapabilities pagingCapabilities, bool recordPermissionCapabilities, int oDataVersion = CurrentODataVersion, bool supportsDataverseOffline = false,
+                                   bool supportsJoinFunction = false)
         {
             Contracts.AssertValueOrNull(sortRestriction);
             Contracts.AssertValueOrNull(filterRestriction);
@@ -120,6 +125,7 @@ public ServiceCapabilities(SortRestriction sortRestriction, FilterRestriction fi
             _columnsCapabilities = null;
             ODataVersion = oDataVersion;
             SupportsRecordPermission = recordPermissionCapabilities;
+            SupportsJoinFunction = supportsJoinFunction;
         }
 
         public static TableDelegationInfo ToDelegationInfo(ServiceCapabilities serviceCapabilities, string tableName, bool isReadOnly, ConnectorType connectorType, string datasetName)
@@ -189,7 +195,10 @@ public static TableDelegationInfo ToDelegationInfo(ServiceCapabilities serviceCa
                 SupportsRecordPermission = serviceCapabilities?.SupportsRecordPermission ?? false,
                 ColumnsCapabilities = columnCapabilities,
                 ColumnsWithRelationships = columnWithRelationships,
-                PrimaryKeyNames = primaryKeyNames
+                PrimaryKeyNames = primaryKeyNames,
+#pragma warning disable CS0618 // Type or member is obsolete
+                SupportsJoinFunction = serviceCapabilities?.SupportsJoinFunction ?? false
+#pragma warning restore CS0618 // Type or member is obsolete
             };
         }
 
@@ -233,14 +242,15 @@ public static ServiceCapabilities ParseTableCapabilities(IDictionary<string, IOp
             string[] filterSupportedFunctions = ParseFilterSupportedFunctions(capabilitiesMetaData);
             PagingCapabilities pagingCapabilities = ParsePagingCapabilities(capabilitiesMetaData);
             bool recordPermissionCapabilities = ParseRecordPermissionCapabilities(capabilitiesMetaData);
+            bool supportsJoinFunction = ParseSupportsJoinCapabilities(capabilitiesMetaData);
             int oDataVersion = capabilitiesMetaData.GetInt(CapabilityConstants.ODataversionOption, defaultValue: CurrentODataVersion);
 
             if (oDataVersion > CurrentODataVersion || oDataVersion < 3)
             {
                 throw new PowerFxConnectorException("Table capabilities specifies an unsupported oDataVersion");
             }
 
-            return new ServiceCapabilities(sortRestriction, filterRestriction, selectionRestriction, groupRestriction, filterFunctions, filterSupportedFunctions, pagingCapabilities, recordPermissionCapabilities, oDataVersion);
+            return new ServiceCapabilities(sortRestriction, filterRestriction, selectionRestriction, groupRestriction, filterFunctions, filterSupportedFunctions, pagingCapabilities, recordPermissionCapabilities, oDataVersion, supportsJoinFunction: supportsJoinFunction);
         }
 
         private static FilterRestriction ParseFilterRestriction(IDictionary<string, IOpenApiAny> capabilitiesMetaData)
@@ -296,5 +306,10 @@ private static bool ParseRecordPermissionCapabilities(IDictionary<string, IOpenA
         {
             return capabilitiesMetaData.GetBool(CapabilityConstants.SupportsRecordPermission);
         }
+
+        private static bool ParseSupportsJoinCapabilities(IDictionary<string, IOpenApiAny> capabilitiesMetaData)
+        {
+            return capabilitiesMetaData.GetBool(CapabilityConstants.SupportsJoin);
+        }
     }
 }

src/libraries/Microsoft.PowerFx.Core/Entities/External/TableDelegationInfo.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.CodeAnalysis;
@@ -22,6 +23,9 @@ public abstract class TableDelegationInfo
         // Used to indicate whether this table has selectable columns
         public SelectionRestrictions SelectionRestriction { get; init; }
 
+        [Obsolete("preview")]
+        public SummarizeCapabilities SummarizeCapabilities { get; init; }
+
         // Defines ungroupable columns
         public GroupRestrictions GroupRestriction { get; init; }
 
@@ -37,6 +41,9 @@ public abstract class TableDelegationInfo
         // Supports per record permission
         internal bool SupportsRecordPermission { get; init; }
 
+        [Obsolete("preview")]
+        public bool SupportsJoinFunction { get; init; }
+
         // Logical name of table
         public string TableName { get; init; }
 
@@ -262,6 +269,35 @@ public SelectionRestrictions()
         }
     }
 
+    [Obsolete("preview")]
+    public class SummarizeCapabilities
+    {
+        public virtual bool IsSummarizableProperty(string propertyName)
+        {
+            return false;
+        }
+
+        public virtual bool IsSummarizableMethod(SummarizeMethod method)
+        {
+            return false;
+        }
+
+        public SummarizeCapabilities()
+        {
+        }
+    }
+
+    [Obsolete("preview")]
+    public enum SummarizeMethod
+    {
+        None,
+        Sum,
+        Average,
+        Min,
+        Max,
+        Count,
+    }
+
     public sealed class FilterRestrictions
     {
         // List of required properties

src/libraries/Microsoft.PowerFx.Core/Functions/Delegation/DelegationStrategies/DelegationValidationStrategy.cs

@@ -424,45 +424,19 @@ protected virtual bool IsValidAsyncOrImpureNode(TexlNode node, TexlBinding bindi
             Contracts.AssertValue(binding);
 
             var isAsync = binding.IsAsync(node);
-            var isPure = binding.IsPure(node);
 
-            if (!isAsync && isPure)
-            {
-                return true;
-            }
-
-            // Async predicates and impure nodes are not supported unless Features say otherwise.
             // Let CallNodes for delegatable async functions be marked as being Valid to allow
             // expressions with delegatable async function calls to be delegated
 
-            // Impure nodes should only be marked valid when Feature is enabled.
-            if (!isPure && !binding.Features.AllowImpureNodeDelegation)
+            if (!isAsync)
             {
-                TrackingProvider.Instance.SetDelegationTrackerStatus(DelegationStatus.ImpureNode, node, binding, trackingFunction ?? Function, DelegationTelemetryInfo.CreateImpureNodeTelemetryInfo(node, binding));
+                return true;
             }
             else
             {
-                if (!isAsync)
-                {
-                    return true;
-                }
-                else if (binding.Features.AllowAsyncDelegation)
-                {
-                    // If the feature is enabled, enable delegation for
-                    // async call, first name and dotted name nodes.
-                    return (node is CallNode) || (node is FirstNameNode) || (node is DottedNameNode);
-                }
-            }
-
-            if (isAsync)
-            {
-                TrackingProvider.Instance.SetDelegationTrackerStatus(DelegationStatus.AsyncPredicate, node, binding, trackingFunction ?? Function, DelegationTelemetryInfo.CreateAsyncNodeTelemetryInfo(node, binding));
+                // Enable delegation for async call, first name, and dotted name nodes.
+                return (node is CallNode) || (node is FirstNameNode) || (node is DottedNameNode);
             }
-
-            var telemetryMessage = string.Format(CultureInfo.InvariantCulture, "Kind:{0}, isAsync:{1}, isPure:{2}", node.Kind, isAsync, isPure);
-            SuggestDelegationHintAndAddTelemetryMessage(node, binding, telemetryMessage);
-
-            return false;
         }
     }
 }

src/libraries/Microsoft.PowerFx.Core/Functions/Delegation/DelegationStrategies/InOpDelegationStrategy.cs

@@ -202,7 +202,7 @@ public override bool IsOpSupportedByTable(OperationCapabilityMetadata metadata,
                 /* Left node can be first name, row scope lambda or a lookup column */
                 (_binaryOpNode.Left.Kind == NodeKind.FirstName || binding.IsFullRecordRowScopeAccess(_binaryOpNode.Left) || (_binaryOpNode.Left.Kind == NodeKind.DottedName && binding.GetType((_binaryOpNode.Left as DottedNameNode).Left).HasExpandInfo)) &&
                 /* Right has to be a single column table */
-                ((_binaryOpNode.Right.Kind == NodeKind.Table || binding.GetType(_binaryOpNode.Right)?.IsColumn == true) && (binding.Features.AllowAsyncDelegation || !binding.IsAsync(_binaryOpNode.Right)));
+                (_binaryOpNode.Right.Kind == NodeKind.Table || binding.GetType(_binaryOpNode.Right)?.IsColumn == true);
 
             if (!(isRHSFirstName || isRHSRecordScope || isCdsInTableDelegation))
             {

src/libraries/Microsoft.PowerFx.Core/Functions/Delegation/DelegationStrategies/OpDelegationStrategy.cs

@@ -99,13 +99,12 @@ private bool IsSupportedNode(TexlNode node, OperationCapabilityMetadata metadata
             // Filter(Accounts, 'Account Name' in ["Foo", Bar"]) - Direct table use
             // Set(Names, ["Foo", Bar"]); Filter(Accounts, 'Account Name' in Names) - Using variable of type table
             // ClearCollect(Names, Accounts); Filter(Accounts, 'Account Name' in Names.'Account Name') - using column from collection.
-            // This won't be delegated if the AllowAsyncDelegation- Filter(Accounts, 'Account Name' in Accounts.'Account Name') as Accounts.'Account Name' is async.
+            // Filter(Accounts, 'Account Name' in Accounts.'Account Name') as Accounts.'Account Name' is async.
             if (isRHSNode
                 && opDelStrategy is BinaryOpDelegationStrategy { Op: BinaryOp.In }
                 && !binding.IsRowScope(node)
                 && binding.GetType(node).IsTable 
                 && binding.GetType(node).IsColumn
-                && (binding.Features.AllowAsyncDelegation || !binding.IsAsync(node))
                 && opDelStrategy.IsOpSupportedByTable(metadata, node, binding))
             {
                 return true;

src/libraries/Microsoft.PowerFx.Core/Public/Config/Features.cs

@@ -48,16 +48,6 @@ public sealed class Features
         /// </summary>
         internal bool RestrictedIsEmptyArguments { get; init; }
 
-        /// <summary>
-        /// Allow delegation for async calls (delegate using awaited call result).
-        /// </summary>
-        internal bool AllowAsyncDelegation { get; init; }
-
-        /// <summary>
-        /// Allow delegation for impure nodes.
-        /// </summary>
-        internal bool AllowImpureNodeDelegation { get; init; }
-
         /// <summary>
         /// Updates the FirstN/LastN functions to require a second argument, instead of
         /// defaulting to 1.
@@ -131,8 +121,6 @@ internal Features(Features other)
             PowerFxV1CompatibilityRules = other.PowerFxV1CompatibilityRules;
             PrimaryOutputPropertyCoercionDeprecated = other.PrimaryOutputPropertyCoercionDeprecated;
             IsUserDefinedTypesEnabled = other.IsUserDefinedTypesEnabled;
-            AllowImpureNodeDelegation = other.AllowImpureNodeDelegation;
-            AllowAsyncDelegation = other.AllowAsyncDelegation;
             AsTypeLegacyCheck = other.AsTypeLegacyCheck;
             JsonFunctionAcceptsLazyTypes = other.JsonFunctionAcceptsLazyTypes;
             IsLookUpReductionDelegationEnabled = other.IsLookUpReductionDelegationEnabled;

src/libraries/Microsoft.PowerFx.Core/Public/Values/DelegationParameters.cs

@@ -73,6 +73,9 @@ public enum DelegationParameterFeatures
         // $apply
         Apply = 1 << 4,
 
+        // $apply = groupby((field1, ..), field with sum as TotalSum)
+        ApplyGroupBy = 1 << 5,
+
         /*
           To be implemented later when needed
          

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems

@@ -247,6 +247,7 @@
     <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetDatasetsMetadata.json" />
     <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetTables.json" />
     <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetSchema ProductModel.json" />
+    <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetSchema Products v2.json" />
     <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetSchema Products.json" />
     <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetTables SampleDB.json" />
     <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL GetRelationships SampleDB.json" />

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs

@@ -270,6 +270,46 @@ public async Task SQL_CdpTabular_GetTables2()
                 string.Join("|", logger.Uris));
         }
 
+        [Fact]
+        public async Task SQL_CdpTabular_JoinCapabilityTest()
+        {
+            using var testConnector = new LoggingTestServer(null /* no swagger */, _output);
+            var config = new PowerFxConfig(Features.PowerFxV1);
+            var engine = new RecalcEngine(config);
+
+            ConsoleLogger logger = new ConsoleLogger(_output);
+            using var httpClient = new HttpClient(testConnector);
+            string connectionId = "2cc03a388d38465fba53f05cd2c76181";
+            string jwt = "eyJ0eXAiOiJKSuA...";
+            using var client = new PowerPlatformConnectorClient("dac64a92-df6a-ee6e-a6a2-be41a923e371.15.common.tip1002.azure-apihub.net", "dac64a92-df6a-ee6e-a6a2-be41a923e371", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" };
+
+            string realTableName = "Product";            
+
+            CdpDataSource cds = new CdpDataSource("default,default");
+
+            testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables SampleDB.json");
+            IEnumerable<CdpTable> tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+
+            CdpTable table = tables.First(t => t.DisplayName == realTableName);
+
+            testConnector.SetResponseFromFiles(@"Responses\SQL GetSchema Products v2.json");
+            await table.InitAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+            Assert.True(table.IsInitialized);
+
+            CdpTableValue sqlTable = table.GetTableValue();
+            Assert.True(sqlTable._tabularService.IsInitialized);
+            Assert.True(sqlTable.IsDelegable);
+
+            HashSet<IExternalTabularDataSource> ads = sqlTable.Type._type.AssociatedDataSources;
+            Assert.NotNull(ads);
+            Assert.Single(ads);
+
+            DataSourceInfo dsi = Assert.IsType<DataSourceInfo>(ads.First());
+#pragma warning disable CS0618 // Type or member is obsolete
+            Assert.True(dsi.DelegationInfo.SupportsJoinFunction);
+#pragma warning restore CS0618 // Type or member is obsolete
+        }
+
         [Fact]
         public async Task SAP_CDP()
         {