Support for $apply/groupby (#1925)

* Implement support for $apply/groupby transformation

Co-authored-by: John Gathogo <jogathogo@microsoft.com>

src/Microsoft.OData.Client/ALinq/ApplyQueryOptionExpression.cs

@@ -32,6 +32,8 @@ internal ApplyQueryOptionExpression(Type type)
         {
             this.Aggregations = new List<Aggregation>();
             this.filterExpressions = new List<Expression>();
+            this.GroupingExpressions = new List<Expression>();
+            this.KeySelectorMap = new Dictionary<string, Expression>(StringComparer.Ordinal);
         }
 
         /// <summary>
@@ -72,6 +74,20 @@ internal Expression GetPredicate()
             return this.filterExpressions.Aggregate((leftExpr, rightExpr) => Expression.And(leftExpr, rightExpr));
         }
 
+        /// <summary>
+        /// The individual expressions that make up the GroupBy selector.
+        /// </summary>
+        internal List<Expression> GroupingExpressions { get; private set; }
+
+        /// <summary>
+        /// Gets a mapping of member names in the GroupBy key selector to their respective expressions.
+        /// </summary>
+        /// <remarks>
+        /// This property will contain a mapping of the member {Name} to the expression {d1.Product.Category.Name} given the following GroupBy expression:
+        /// dsc.CreateQuery&lt;Sale&gt;("Sales").GroupBy(d1 => d1.Product.Category.Name, (d2, d3) => new { CategoryName = d2, AverageAmount = d3.Average(d4 => d4.Amount) })
+        /// </remarks>
+        internal Dictionary<string, Expression> KeySelectorMap { get; private set; }
+
         /// <summary>
         /// Structure for an aggregation. Holds lambda expression plus enum indicating aggregation method
         /// </summary>

src/Microsoft.OData.Client/ALinq/DataServiceQueryProvider.cs

@@ -172,9 +172,28 @@ internal QueryComponents Translate(Expression e)
 
             UriWriter.Translate(this.Context, addTrailingParens, e, out uri, out version);
             ResourceExpression re = e as ResourceExpression;
-            Type lastSegmentType = re.Projection == null ? re.ResourceType : re.Projection.Selector.Parameters[0].Type;
+            ApplyQueryOptionExpression applyQueryOptionExpr = (re as QueryableResourceExpression)?.Apply;
+            Type lastSegmentType;
+
+            // The KeySelectorMap property is initialized and populated with a least one item if we're dealing with a GroupBy expression.
+            // In that case, the selector in the Projection will take the following form:
+            // (d2, d3) => new <>f_AnonymousType13`2(CategoryName = d2, AverageAmount = d3.Average(d4 => d4))
+            // We examine the 2nd parameter to determine the type of values in the IGrouping<TKey, TElement>
+            // The TElement type implements IEnumerable and the first generic argument should be our last segment type
+            if (applyQueryOptionExpr?.KeySelectorMap?.Count > 0)
+            {
+                lastSegmentType = re.Projection.Selector.Parameters[1].Type.GetGenericArguments()[0];
+            }
+            else
+            {
+                lastSegmentType = re.Projection == null ? re.ResourceType : re.Projection.Selector.Parameters[0].Type;
+            }
+
             LambdaExpression selector = re.Projection == null ? null : re.Projection.Selector;
-            return new QueryComponents(uri, version, lastSegmentType, selector, normalizerRewrites);
+            QueryComponents queryComponents = new QueryComponents(uri, version, lastSegmentType, selector, normalizerRewrites);
+            queryComponents.GroupByKeySelectorMap = applyQueryOptionExpr?.KeySelectorMap;
+
+            return queryComponents;
         }
 
         /// <summary>

src/Microsoft.OData.Client/ALinq/QueryComponents.cs

@@ -223,6 +223,15 @@ internal bool HasSelectQueryOption
         /// <value> URI with additional query options added if required. </value>
         internal Uri Uri { get; set; }
 
+        /// <summary>
+        /// Gets or sets a mapping of member names in the GroupBy key selector to their respective expressions.
+        /// </summary>
+        /// <remarks>
+        /// This property will contain a mapping of the member {Name} to the expression {d1.Product.Category.Name} given the following GroupBy expression:
+        /// dsc.CreateQuery&lt;Sale&gt;("Sales").GroupBy(d1 => d1.Product.Category.Name, (d2, d3) => new { CategoryName = d2, AverageAmount = d3.Average(d4 => d4.Amount) })
+        /// </remarks>
+        internal Dictionary<string, Expression> GroupByKeySelectorMap { get; set; }
+
         /// <summary>
         /// Determines whether or not the specified query string contains the $select query option.
         /// </summary>

src/Microsoft.OData.Client/ALinq/QueryableResourceExpression.cs

@@ -13,6 +13,7 @@ namespace Microsoft.OData.Client
     using System.Linq;
     using System.Linq.Expressions;
     using System.Reflection;
+    using Microsoft.OData.UriParser.Aggregation;
 
     /// <summary>Queryable Resource Expression, the base class for ResourceSetExpression and SingletonExpression</summary>
     [DebuggerDisplay("QueryableResourceExpression {Source}.{MemberExpression}")]
@@ -371,8 +372,24 @@ internal void AddFilter(IEnumerable<Expression> predicateConjuncts)
             this.keyPredicateConjuncts.Clear();
         }
 
-        internal void AddApply(Expression aggregateExpr, OData.UriParser.Aggregation.AggregationMethod aggregationMethod)
+        /// <summary>
+        /// Adds an aggregation expression to the $apply expression of this resource expression.
+        /// If the resource expression already contains a $filter expression, 
+        /// the predicate conjuncts of the $filter expression are moved
+        /// to the predicate conjuncts of the $apply expression and 
+        /// the $filter expression removed.
+        /// The predicate conjuncts in the $apply expression are used to generate 
+        /// a filter transformation used to restrict the set of data to be aggregated.
+        /// </summary>
+        /// <param name="aggregationExpr">The aggregation expression.</param>
+        /// <param name="aggregationMethod">The aggregation method.</param>
+        /// <param name="aggregationAlias">The aggregation alias.</param>
+        internal void AddAggregation(Expression aggregationExpr, AggregationMethod aggregationMethod, string aggregationAlias = "")
         {
+            // Aggregation expression should not be null if the aggregation method is anything other than VirtualPropertyCount ($count)
+            Debug.Assert(aggregationExpr != null || aggregationMethod == AggregationMethod.VirtualPropertyCount,
+                "aggregationExpr != null || aggregationMethod == AggregationMethod.VirtualPropertyCount");
+
             if (this.OrderBy != null)
             {
                 throw new NotSupportedException(Strings.ALinq_QueryOptionOutOfOrder("apply", "orderby"));
@@ -400,12 +417,12 @@ internal void AddApply(Expression aggregateExpr, OData.UriParser.Aggregation.Agg
                 // The $apply query option is evaluated first, then other query options ($filter, $orderby, $select) are evaluated, 
                 // if applicable, on the result of $apply in their normal order.
                 // http://docs.oasis-open.org/odata/odata-data-aggregation-ext/v4.0/cs02/odata-data-aggregation-ext-v4.0-cs02.html#_Toc435016590
-                
+
                 // If a Where appears before an aggregation method (e.g. Average, Sum, etc) or GroupBy, 
                 // the conjuncts of the filter expression will be used to restrict the set of data to be aggregated. 
                 // They will not appear on the $filter query option. Instead, we use them to construct a filter transformation.
                 // E.g. /Sales?$apply=filter(Amount gt 1)/aggregate(Amount with average as AverageAmount)
-                
+
                 // If a Where appears after an aggregation method or GroupBy, the conjuncts should appear
                 // on a $filter query option after the $apply.
                 // E.g. /Sales?$apply=groupby((Product/Color),aggregate(Amount with average as AverageAmount))&$filter=Product/Color eq 'Brown'
@@ -418,7 +435,7 @@ internal void AddApply(Expression aggregateExpr, OData.UriParser.Aggregation.Agg
                 this.RemoveFilterExpression();
             }
 
-            this.Apply.Aggregations.Add(new ApplyQueryOptionExpression.Aggregation(aggregateExpr, aggregationMethod));
+            this.Apply.Aggregations.Add(new ApplyQueryOptionExpression.Aggregation(aggregationExpr, aggregationMethod, aggregationAlias));
         }
 
         /// <summary>

src/Microsoft.OData.Client/ALinq/ResourceBinder.cs

@@ -831,11 +831,64 @@ private static Expression AnalyzeAggregation(MethodCallExpression aggregationExp
                 return aggregationExpr;
             }
 
-            resourceExpr.AddApply(selector, aggregationMethod);
+            resourceExpr.AddAggregation(selector, aggregationMethod);
 
             return resourceExpr;
         }
 
+        /// <summary>
+        /// Analyzes a GroupBy(source, keySelector, resultSelector) method expression to 
+        /// determine whether it can be satisfied with $apply.
+        /// </summary>
+        /// <param name="methodCallExpr">Expression to analyze.</param>
+        /// <param name="expr">
+        /// <paramref name="methodCallExpr"/>, or a new resource set expression
+        /// for the target resource in the analyzed expression.
+        /// </param>
+        /// <returns>true if <paramref name="methodCallExpr"/> can be satisfied with $apply; false otherwise.</returns>
+        private bool AnalyzeGroupBy(MethodCallExpression methodCallExpr, out Expression expr)
+        {
+            Debug.Assert(methodCallExpr != null, "methodCallExpr != null");
+
+            expr = methodCallExpr;
+
+            // Total 3 arguments expected
+            // GroupBy<TSource, TKey, TResult>(
+            //     IQueryable<TSource> source,
+            //     Expression<Func<TSource, TKey>> keySelector,
+            //     Expression<Func<TKey, IEnumerable<TSource>, TResult>> resultSelector)
+            if (methodCallExpr.Arguments.Count != 3)
+            {
+                return false;
+            }
+
+            SequenceMethod sequenceMethod;
+
+            if (!ReflectionUtil.TryIdentifySequenceMethod(methodCallExpr.Method, out sequenceMethod) ||
+                sequenceMethod != SequenceMethod.GroupByResultSelector)
+            {
+                return false;
+            }
+
+            // We expect a resource expression as the first argument.
+            QueryableResourceExpression source = this.Visit(methodCallExpr.Arguments[0]) as QueryableResourceExpression;
+            if (source == null)
+            {
+                return false;
+            }
+
+            ResourceExpression resourceExpr = source.CreateCloneWithNewType(methodCallExpr.Method.ReturnType);
+
+            if (!GroupByAnalyzer.Analyze(methodCallExpr, source, resourceExpr))
+            {
+                return false;
+            }
+
+            expr = resourceExpr;
+
+            return true;
+        }
+
         /// <summary>Ensures that there's a limit on the cardinality of a query.</summary>
         /// <param name="mce"><see cref="MethodCallExpression"/> for the method to limit First/Single(OrDefault).</param>
         /// <param name="maxCardinality">Maximum cardinality to allow.</param>
@@ -1308,6 +1361,7 @@ internal override Expression VisitQueryableResourceExpression(QueryableResourceE
                 // since we could be adding query options to the root, create a new one which can be mutable.
                 return QueryableResourceExpression.CreateNavigationResourceExpression(rse.NodeType, rse.Type, rse.Source, rse.MemberExpression, rse.ResourceType, null /*expandPaths*/, CountOption.None, null /*customQueryOptions*/, null /*projection*/, rse.ResourceTypeAs, rse.UriVersion, rse.OperationName, rse.OperationParameters);
             }
+
             return rse;
         }
 
@@ -1373,7 +1427,7 @@ private static Expression AnalyzeCountMethod(MethodCallExpression mce)
             return rseCopy;
         }
 
-        private static void AddSequenceQueryOption(ResourceExpression target, QueryOptionExpression qoe)
+        internal static void AddSequenceQueryOption(ResourceExpression target, QueryOptionExpression qoe)
         {
             QueryableResourceExpression rse = (QueryableResourceExpression)target;
             rse.ConvertKeyToFilterExpression();
@@ -1464,13 +1518,16 @@ internal override Expression VisitMethodCall(MethodCallExpression mce)
             {
                 // The leaf projection can be one of Select(source, selector) or
                 // SelectMany(source, collectionSelector, resultSelector).
-                if (sequenceMethod == SequenceMethod.Select ||
-                    sequenceMethod == SequenceMethod.SelectManyResultSelector)
+                if ((sequenceMethod == SequenceMethod.Select ||
+                    sequenceMethod == SequenceMethod.SelectManyResultSelector) &&
+                    this.AnalyzeProjection(mce, sequenceMethod, out e))
                 {
-                    if (this.AnalyzeProjection(mce, sequenceMethod, out e))
-                    {
-                        return e;
-                    }
+                    return e;
+                }
+                else if (sequenceMethod == SequenceMethod.GroupByResultSelector &&
+                    this.AnalyzeGroupBy(mce, out e))
+                {
+                    return e;
                 }
             }
 
@@ -3032,6 +3089,46 @@ internal static void ValidateAggregateExpression(Expression expr)
                     }
                 }
             }
+
+            /// <summary>
+            /// Checks whether the specified <paramref name="expr"/> is a valid grouping expression.
+            /// </summary>
+            /// <param name="expr">The grouping expression</param>
+            internal static void ValidateGroupingExpression(Expression expr)
+            {
+                MemberExpression memberExpr = StripTo<MemberExpression>(expr);
+                Debug.Assert(memberExpr != null, "memberExpr != null");
+
+                // NOTE: Based on the spec, if the property path leads to a single-valued navigation 
+                // property, this means grouping by the entity-id of the related entities.
+                // However, that support is not implemented in OData WebApi. At the moment, grouping 
+                // expression must evaluate to a single-valued primitive property
+
+                // Disallow unsupported scenarios like the following:
+                // - GroupBy(d1 => d1.Property.Length)
+                // - GroupBy(d1 => d1.CollectionProperty.Count)
+                MemberExpression containingExpr = StripTo<MemberExpression>(memberExpr.Expression);
+                if (containingExpr != null)
+                {
+                    if (PrimitiveType.IsKnownNullableType(containingExpr.Type))
+                    {
+                        throw new NotSupportedException(Strings.ALinq_InvalidGroupingExpression(memberExpr));
+                    }
+
+                    Type collectionType = ClientTypeUtil.GetImplementationType(containingExpr.Type, typeof(ICollection<>));
+                    if (collectionType != null)
+                    {
+                        throw new NotSupportedException(Strings.ALinq_InvalidGroupingExpression(memberExpr));
+                    }
+                }
+
+                // Disallow grouping expressions that evaluate to a single-valued complex or navigation property
+                // Due to feature gap in OData WebApi
+                if (!PrimitiveType.IsKnownNullableType(memberExpr.Type))
+                {
+                    throw new NotSupportedException(Strings.ALinq_InvalidGroupingExpression(memberExpr));
+                }
+            }
         }
 
         // TODO: By default, C#/Vb compilers uses declaring type for property expression when