From 07b64d987a772c0499fdad340592ea5452bef64f Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Tue, 23 Feb 2021 12:23:53 -0800 Subject: [PATCH 01/24] MetadataGenerator adds cardinality-many and value provider understands collections event hubs e2e initial updates temp add to solution too update to new model rebase to latest Improved logic for resolving generics remove webjobs reference Fix IEnumerable some enumerable converter tests Clean up test Cleanup reverting local settings json change change localsettings typo added datatype string to some tests fix generator test --- sdk/Sdk/DataTypeEnum.cs | 14 + sdk/Sdk/FunctionMetadataGenerator.cs | 207 +++++++++- .../Converters/EnumerableConverter.cs | 95 +++++ .../Hosting/ServiceCollectionExtensions.cs | 3 +- .../Converters/EnumerableConverterTests.cs | 82 ++++ test/E2ETests/E2EApps/E2EApp/E2EApp.csproj | 1 + .../EventHubs/EventHubsEnumerableFunctions.cs | 46 +++ .../EventHubs/EventHubsObjectFunctions.cs | 43 +++ .../EventHubs/EventHubsStringFunctions.cs | 39 ++ .../E2EApps/E2EApp/EventHubs/TestData.cs | 23 ++ .../E2EApps/E2EApp/Http/BasicHttpFunctions.cs | 2 - .../E2EApps/E2EApp/local.settings.json | 3 +- test/E2ETests/E2ETests/E2ETests.sln | 6 + .../FunctionMetadataGeneratorTests.cs | 355 +++++++++++++++++- .../SdkTests.csproj | 1 + 15 files changed, 904 insertions(+), 16 deletions(-) create mode 100644 sdk/Sdk/DataTypeEnum.cs create mode 100644 src/DotNetWorker/Converters/EnumerableConverter.cs create mode 100644 test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs create mode 100644 test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs create mode 100644 test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs create mode 100644 test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs create mode 100644 test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs diff --git a/sdk/Sdk/DataTypeEnum.cs b/sdk/Sdk/DataTypeEnum.cs new file mode 100644 index 000000000..9a3ad85d2 --- /dev/null +++ b/sdk/Sdk/DataTypeEnum.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Azure.Functions.Worker.Sdk +{ + internal enum DataType + { + Undefined, + Binary, + String, + Stream + } +} diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index e28dc3fdb..84cbdb9c0 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Dynamic; @@ -23,6 +24,11 @@ internal class FunctionMetadataGenerator private const string VoidType = "System.Void"; private const string ReturnBindingName = "$return"; private const string HttpTriggerBindingType = "HttpTrigger"; + private const string IEnumerableOfStringType = "System.Collections.Generic.IEnumerable`1"; + private const string IEnumerableOfBinaryType = "System.Collections.Generic.IEnumerable`1"; + private const string IEnumerableOfT = "System.Collections.Generic.IEnumerable`1"; + private const string IEnumerableOfKeyValuePair = "System.Collections.Generic.IEnumerable`1>"; + private const string GenericIEnumerableArgumentName = "T"; private readonly IndentableLogger _logger; @@ -324,7 +330,7 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe { if (IsFunctionBindingType(parameterAttribute)) { - AddBindingMetadata(bindingMetadata, parameterAttribute, parameter.Name); + AddBindingMetadata(bindingMetadata, parameterAttribute, parameter.Name, parameter.ParameterType); AddExtensionInfo(_extensions, parameterAttribute); } } @@ -353,18 +359,18 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe private static void AddOutputBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? name = null) { - AddBindingMetadata(bindingMetadata, attribute, parameterName: name); + AddBindingMetadata(bindingMetadata, attribute, parameterName: name, parameterType: null); } - private static void AddBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? parameterName) + private static void AddBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? parameterName, TypeReference? parameterType) { string bindingType = GetBindingType(attribute); - ExpandoObject binding = BuildBindingMetadataFromAttribute(attribute, bindingType, parameterName); + ExpandoObject binding = BuildBindingMetadataFromAttribute(attribute, bindingType, parameterName, parameterType); bindingMetadata.Add(binding); } - private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute attribute, string bindingType, string? parameterName) + private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute attribute, string bindingType, string? parameterName, TypeReference? parameterType) { ExpandoObject binding = new ExpandoObject(); @@ -378,6 +384,40 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict["Type"] = bindingType; bindingDict["Direction"] = GetBindingDirection(attribute); + // Inspect parameter type + if (parameterType is not null) + { + // Is string parameter type + if (IsStringType(parameterType.FullName)) + { + bindingDict["DataType"] = "String"; + } + + // Is binary parameter type + if (IsBinaryType(parameterType.FullName)) + { + bindingDict["DataType"] = "Binary"; + } + + // Trigger logic + if (bindingType.EndsWith("Trigger", StringComparison.OrdinalIgnoreCase)) + { + // Add "cardinality": "many" if we see an IEnumerable type or array type + if (IsIterableCollection(parameterType, out DataType dataType)) + { + bindingDict["Cardinality"] = "Many"; + if (dataType.Equals(DataType.String)) + { + bindingDict["DataType"] = "String"; + } + else if (dataType.Equals(DataType.Binary)) + { + bindingDict["DataType"] = "Binary"; + } + } + } + } + foreach (var property in attribute.GetAllDefinedProperties()) { bindingDict.Add(property.Key, property.Value); @@ -386,6 +426,163 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a return binding; } + private static bool IsIterableCollection(TypeReference type, out DataType dataType) + { + // Array and not byte array + bool isArray = type.IsArray && !string.Equals(type.FullName, typeof(byte[]).FullName, StringComparison.Ordinal); + if (isArray) + { + TypeSpecification? typeSpecification = type as TypeSpecification; + if (typeSpecification is not null) + { + dataType = GetDataTypeFromType(typeSpecification.ElementType.FullName); + return true; + } + } + + bool isMappingEnumerable = IsOrDerivedFrom(type, IEnumerableOfKeyValuePair) + || IsOrDerivedFrom(type, typeof(Lookup<,>).FullName) + || IsOrDerivedFrom(type, typeof(Dictionary<,>).FullName); + if (isMappingEnumerable) + { + dataType = DataType.Undefined; + return false; + } + + // IEnumerable and not string or dictionary + bool isEnumerableOfT = IsOrDerivedFrom(type, IEnumerableOfT); + bool isEnumerableCollection = + !IsStringType(type.FullName) + && (IsOrDerivedFrom(type, typeof(IEnumerable).FullName) + || IsOrDerivedFrom(type, typeof(IEnumerable<>).FullName) + || isEnumerableOfT); + if (isEnumerableCollection) + { + dataType = DataType.Undefined; + if (IsOrDerivedFrom(type, IEnumerableOfStringType)) + { + dataType = DataType.String; + } + else if (IsOrDerivedFrom(type, IEnumerableOfBinaryType)) + { + dataType = DataType.Binary; + } + else if (isEnumerableOfT) + { + // Find real type that "T" in IEnumerable resolves to + string typeName = ResolveIEnumerableOfTType(type, new Dictionary()) ?? string.Empty; + dataType = GetDataTypeFromType(typeName); + } + return true; + } + + dataType = DataType.Undefined; + return false; + } + + private static bool IsOrDerivedFrom(TypeReference type, string interfaceFullName) + { + bool isType = string.Equals(type.FullName, interfaceFullName, StringComparison.Ordinal); + TypeDefinition definition = type.Resolve(); + return isType || IsDerivedFrom(definition, interfaceFullName); + } + + private static bool IsDerivedFrom(TypeDefinition definition, string interfaceFullName) + { + var isType = string.Equals(definition.FullName, interfaceFullName, StringComparison.Ordinal); + return isType || HasInterface(definition, interfaceFullName) || IsSubclassOf(definition, interfaceFullName); + } + + private static bool HasInterface(TypeDefinition definition, string interfaceFullName) + { + return definition.Interfaces.Any(i => string.Equals(i.InterfaceType.FullName, interfaceFullName, StringComparison.Ordinal)) + || definition.NestedTypes.Any(t => IsDerivedFrom(t, interfaceFullName)); + } + + private static bool IsSubclassOf(TypeDefinition definition, string interfaceFullName) + { + if (definition.BaseType is null) + { + return false; + } + + TypeDefinition baseType = definition.BaseType.Resolve(); + return IsDerivedFrom(baseType, interfaceFullName); + } + + private static string? ResolveIEnumerableOfTType(TypeReference type, Dictionary foundMapping) + { + // Base case: + // We are at IEnumerable and want to return the most recent resolution of T + // (Most recent is relative to IEnumerable) + if (string.Equals(type.FullName, IEnumerableOfT, StringComparison.Ordinal)) + { + if (foundMapping.TryGetValue(GenericIEnumerableArgumentName, out string typeName)) + { + return typeName; + } + else + { + return null; + } + } + + TypeDefinition definition = type.Resolve(); + if (definition.HasGenericParameters && type is GenericInstanceType genericType) + { + for (int i = 0; i < genericType.GenericArguments.Count(); i++) + { + string name = genericType.GenericArguments.ElementAt(i).FullName; + string resolvedName = definition.GenericParameters.ElementAt(i).FullName; + + if (foundMapping.TryGetValue(name, out string firstType)) + { + foundMapping.Remove(name); + foundMapping.Add(resolvedName, firstType); + } + else + { + foundMapping.Add(resolvedName, name); + } + } + + } + + return definition.Interfaces + .Select(i => ResolveIEnumerableOfTType(i.InterfaceType, foundMapping)) + .Where(name => name is not null) + .FirstOrDefault() + ?? definition.NestedTypes + .Select(t => ResolveIEnumerableOfTType(t, foundMapping)) + .Where(name => name is not null) + .FirstOrDefault() + ?? ResolveIEnumerableOfTType(definition.BaseType, foundMapping); + } + + private static DataType GetDataTypeFromType(string fullName) + { + if (IsStringType(fullName)) + { + return DataType.String; + } + else if (IsBinaryType(fullName)) + { + return DataType.Binary; + } + + return DataType.Undefined; + } + + private static bool IsStringType(string fullName) + { + return string.Equals(fullName, typeof(string).FullName, StringComparison.Ordinal); + } + + private static bool IsBinaryType(string fullName) + { + return string.Equals(fullName, typeof(byte[]).FullName, StringComparison.Ordinal); + } + private static string GetBindingType(CustomAttribute attribute) { var attributeType = attribute.AttributeType.Name; diff --git a/src/DotNetWorker/Converters/EnumerableConverter.cs b/src/DotNetWorker/Converters/EnumerableConverter.cs new file mode 100644 index 000000000..344be1fbe --- /dev/null +++ b/src/DotNetWorker/Converters/EnumerableConverter.cs @@ -0,0 +1,95 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Azure.Functions.Worker.Converters +{ + // Converting IEnumerable<> to Array + internal class EnumerableConverter : IConverter + { + private static Type ListType = typeof(List<>); + private static Type HashSetType = typeof(HashSet<>); + + // Convert IEnumerable from common types. IEnumerable types will + // be converted by TypeConverter. + public bool TryConvert(ConverterContext context, out object? target) + { + EnumerableTargetType? targetType = null; + // Array + if (context.Parameter.Type.IsArray) + { + targetType = EnumerableTargetType.Array; + } + // List or HashSet + else if (context.Parameter.Type.IsGenericType) + { + if (context.Parameter.Type.GetGenericTypeDefinition().IsAssignableFrom(ListType)) + { + targetType = EnumerableTargetType.List; + } + else if (context.Parameter.Type.GetGenericTypeDefinition().IsAssignableFrom(HashSetType)) + { + targetType = EnumerableTargetType.HashSet; + } + } + + // Only apply if user is requesting an array, list, or hashset + if (targetType is not null) + { + // Valid options from FunctionRpc.proto are string, byte, double and long collection + if (context.Source is IEnumerable enumerableString) + { + target = GetTarget(enumerableString, targetType); + return true; + } + else if (context.Source is IEnumerable enumerableBytes) + { + target = GetTarget(enumerableBytes, targetType); + return true; + } + else if (context.Source is IEnumerable enumerableDouble) + { + target = GetTarget(enumerableDouble, targetType); + return true; + } + else if (context.Source is IEnumerable enumerableLong) + { + target = GetTarget(enumerableLong, targetType); + return true; + } + } + + target = default; + return false; + } + + // Dictionary and Lookup not handled because we don't know + // what they keySelector and elementSelector should be. + private static object? GetTarget(IEnumerable source, EnumerableTargetType? targetType) + { + switch (targetType) + { + case EnumerableTargetType.Array: + return source.ToArray(); + case EnumerableTargetType.HashSet: + return source.ToHashSet(); + case EnumerableTargetType.List: + return source.ToList(); + default: + return null; + } + } + + private enum EnumerableTargetType + { + Array, + List, + Dictionary, + HashSet, + Lookup + } + } +} diff --git a/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs b/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs index 2aa25778a..6b8fef7cc 100644 --- a/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs +++ b/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs @@ -109,7 +109,8 @@ internal static IServiceCollection RegisterDefaultConverters(this IServiceCollec .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); } internal static IServiceCollection RegisterOutputChannel(this IServiceCollection services) diff --git a/test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs b/test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs new file mode 100644 index 000000000..0c5b27716 --- /dev/null +++ b/test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf.Collections; +using Microsoft.Azure.Functions.Worker.Converters; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Tests.Converters +{ + public class EnumerableConverterTests + { + private const string _sourceString = "hello"; + private static readonly byte[] _sourceBytes = Encoding.UTF8.GetBytes(_sourceString); + private static readonly ReadOnlyMemory _sourceMemory = new ReadOnlyMemory(_sourceBytes); + private EnumerableConverter _converter = new EnumerableConverter(); + + private static readonly IEnumerable _sourceMemoryEnumerable = new RepeatedField() { _sourceBytes }; + private static readonly RepeatedField _sourceStringEnumerable = new RepeatedField() { _sourceString }; + private static readonly RepeatedField _sourceDoubleEnumerable = new RepeatedField() { 1.0 }; + private static readonly RepeatedField _sourceLongEnumerable = new RepeatedField() { 2000 }; + + [Fact] + public void ConvertCollectionBytesToByteDoubleArray() + { + var context = new TestConverterContext("output", typeof(byte[][]), _sourceMemoryEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + + [Fact] + public void ConvertCollectionBytesToByteArrayList() + { + var context = new TestConverterContext("output", typeof(List), _sourceMemoryEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert>(target); + } + + [Fact] + public void ConvertCollectionBytesToByteArrayHashSet() + { + var context = new TestConverterContext("output", typeof(HashSet), _sourceMemoryEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert>(target); + } + + // Queue implements IEnumerable but is not converted + [Fact] + public void ConvertCollectionBytesToByteArrayQueue() + { + var context = new TestConverterContext("output", typeof(Queue), _sourceMemoryEnumerable); + Assert.False(context.Parameter.Type.IsAssignableFrom(typeof(IEnumerable))); + Assert.False(_converter.TryConvert(context, out object target)); + } + + [Fact] + public void ConvertCollectionStringToStringArray() + { + var context = new TestConverterContext("output", typeof(string[]), _sourceStringEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + + [Fact] + public void ConvertCollectionSint64ToLongArray() + { + var context = new TestConverterContext("output", typeof(long[]), _sourceLongEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + + [Fact] + public void ConvertCollectionDoubleToDoubleArray() + { + var context = new TestConverterContext("output", typeof(double[]), _sourceDoubleEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj index 245e53bdb..1f11d3b95 100644 --- a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj +++ b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj @@ -14,6 +14,7 @@ + diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs new file mode 100644 index 000000000..1e3b35030 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using Microsoft.Azure.Functions.Worker.E2EApp.EventHubs; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp +{ + public static class EventHubsEnumerableFunctions + { + [Function(nameof(EventHubsEnumerableTrigger))] + [EventHubOutput("test-output-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData EventHubsEnumerableTrigger([EventHubTrigger("test-input-enumerable-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsEnumerableTrigger)); + logger.LogInformation($"First trigger (List)!!"); + input.ForEach(item => logger.LogInformation(item.ToString())); + return input[0]; + } + + [Function(nameof(EventHubsVerifyOutputEnumerable))] + [QueueOutput("test-eventhub-output-string-dotnet-isolated")] + public static string EventHubsVerifyOutputEnumerable([EventHubTrigger("test-output-enumerable-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsVerifyOutputEnumerable)); + logger.LogInformation($"Second trigger (List)!! '{input[0]}'"); + return input[0]; + } + + [Function(nameof(TestEnumerable))] + [EventHubOutput("test-input-enumerable-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData TestEnumerable( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestData req, + FunctionContext context) + { + var logger = context.GetLogger(nameof(TestEnumerable)); + logger.LogInformation(".NET Worker HTTP trigger function processed a request"); + return new TestData() + { + Name = "Ballmer", + TimeProperty = "2021-01-27T15:57:38.000-06:00" + }; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs new file mode 100644 index 000000000..d288a4b5b --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs @@ -0,0 +1,43 @@ +using Microsoft.Azure.Functions.Worker.E2EApp.EventHubs; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp +{ + public static class EventHubsObjectFunctions + { + [Function(nameof(EventHubsObjectFunction))] + [EventHubOutput("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData EventHubsObjectFunction([EventHubTrigger("test-eventhub-input-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] TestData input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsObjectFunction)); + logger.LogInformation($"First Trigger (TestData)!! Name: '{input.Name}' Time: '{input.TimeProperty}'"); + return input; + } + + [Function(nameof(EventHubsVerifyOutputObject))] + [QueueOutput("test-eventhub-output-object-dotnet-isolated")] + public static TestData EventHubsVerifyOutputObject([EventHubTrigger("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] TestData input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsVerifyOutputObject)); + logger.LogInformation($"Second Trigger (TestData)!! Name: '{input.Name}' Time: '{input.TimeProperty}'"); + return input; + } + + [Function(nameof(TestObject))] + [EventHubOutput("test-eventhub-input-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData TestObject( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestData req, + FunctionContext context) + { + var logger = context.GetLogger(nameof(TestObject)); + return new TestData() + { + Name = "Ballmer", + TimeProperty = "2021-01-27T15:57:38.000-09:00" + }; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs new file mode 100644 index 000000000..09c2143fb --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs @@ -0,0 +1,39 @@ +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp +{ + public static class EventHubsStringFunctions + { + [Function(nameof(EventHubsStringTrigger))] + [EventHubOutput("test-output-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static string EventHubsStringTrigger([EventHubTrigger("test-input-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsStringTrigger)); + logger.LogInformation($"First trigger (string[])!! '{input[0]}'"); + return input[0]; + } + + [Function(nameof(EventHubsVerifyOutputString))] + [QueueOutput("test-eventhub-output-string-dotnet-isolated")] + public static string EventHubsVerifyOutputString([EventHubTrigger("test-output-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsVerifyOutputString)); + logger.LogInformation($"Second trigger (string[])!! '{input[0]}'"); + return input[0]; + } + + [Function(nameof(Test))] + [EventHubOutput("test-input-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static string Test( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestData req, + FunctionContext context) + { + var logger = context.GetLogger(nameof(Test)); + logger.LogInformation(".NET Worker HTTP trigger function processed a request"); + return "hello world"; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs new file mode 100644 index 000000000..3daa270b4 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Functions.Worker.E2EApp.EventHubs +{ + public class TestData + { + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("TimeProperty")] + public string TimeProperty { get; set; } + + public override string ToString() + { + return $"Name: {Name}, TimeProperty: {TimeProperty}"; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs index 4ab45a68f..aee020dd0 100644 --- a/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs @@ -2,12 +2,10 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System.Net; -using System.Text; using System.Text.Json; using System.Web; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; -using Microsoft.Azure.Functions.Worker.Pipeline; using Microsoft.Extensions.Logging; namespace Microsoft.Azure.Functions.Worker.E2EApp diff --git a/test/E2ETests/E2EApps/E2EApp/local.settings.json b/test/E2ETests/E2EApps/E2EApp/local.settings.json index 7e1a57612..074751b6b 100644 --- a/test/E2ETests/E2EApps/E2EApp/local.settings.json +++ b/test/E2ETests/E2EApps/E2EApp/local.settings.json @@ -6,6 +6,7 @@ "CosmosConnection": "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "CosmosDb": "ItemDb", "CosmosCollIn": "ItemCollectionIn", - "CosmosCollOut": "ItemCollectionOut" + "CosmosCollOut": "ItemCollectionOut", + "EventHubConnectionAppSetting": "%EventHubConnectionAppSetting%" } } \ No newline at end of file diff --git a/test/E2ETests/E2ETests/E2ETests.sln b/test/E2ETests/E2ETests/E2ETests.sln index 55443eaa7..1a3e1ac90 100644 --- a/test/E2ETests/E2ETests/E2ETests.sln +++ b/test/E2ETests/E2ETests/E2ETests.sln @@ -23,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.CosmosDB" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.Storage", "..\..\..\extensions\Worker.Extensions.Storage\Worker.Extensions.Storage.csproj", "{34894B4F-B58C-49E8-84D7-D6AACA200B06}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.EventHubs", "..\..\..\extensions\Worker.Extensions.EventHubs\Worker.Extensions.EventHubs.csproj", "{09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -61,6 +63,10 @@ Global {34894B4F-B58C-49E8-84D7-D6AACA200B06}.Debug|Any CPU.Build.0 = Debug|Any CPU {34894B4F-B58C-49E8-84D7-D6AACA200B06}.Release|Any CPU.ActiveCfg = Release|Any CPU {34894B4F-B58C-49E8-84D7-D6AACA200B06}.Release|Any CPU.Build.0 = Release|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 82ffe81f8..70aa097f0 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Dynamic; using System.Linq; @@ -109,7 +111,7 @@ public void BasicHttpFunctionWithExternalReturnType() }); ValidateFunction(functions.Single(), ExternalType_Return.FunctionName, GetEntryPoint(nameof(ExternalType_Return), nameof(ExternalType_Return.Http)), - b => ValidateTrigger(b), + b => ValidateTrigger(b), b => ValidateQueueOutput(b)); void ValidateTrigger(ExpandoObject b) @@ -169,7 +171,8 @@ void ValidateQueueTrigger(ExpandoObject b) { "Type", "QueueTrigger" }, { "Direction", "In" }, { "Connection", "MyConnection" }, - { "queueName", "queueName" } + { "queueName", "queueName" }, + { "DataType", "String" } }); } @@ -196,7 +199,8 @@ void ValidateBlobTrigger(ExpandoObject b) { "Name", "blob" }, { "Type", "BlobTrigger" }, { "Direction", "In" }, - { "blobPath", "container2/%file%" } + { "blobPath", "container2/%file%" }, + { "DataType", "String" } }); } @@ -270,7 +274,8 @@ void ValidateQueueTrigger(ExpandoObject b) { "Type", "QueueTrigger" }, { "Direction", "In" }, { "Connection", "MyConnection" }, - { "queueName", "queueName" } + { "queueName", "queueName" }, + { "DataType", "String" } }); } @@ -328,7 +333,8 @@ void ValidateHttpTrigger(ExpandoObject b) { "Name", "req" }, { "Type", "HttpTrigger" }, { "Direction", "In" }, - { "methods", new[] { "get" } } + { "methods", new[] { "get" } }, + { "DataType", "String" } }); } @@ -380,7 +386,8 @@ void ValidateHttpTrigger(ExpandoObject b) { "Name", "req" }, { "Type", "HttpTrigger" }, { "Direction", "In" }, - { "methods", new[] { "get" } } + { "methods", new[] { "get" } }, + { "DataType", "String" } }); } @@ -407,6 +414,75 @@ public void MultiOutput_OnMethod_Throws() Assert.Contains($"Found multiple Output bindings on method", exception.Message); } + [Theory] + [InlineData("StringInputFunction", nameof(CardinalityMany.StringInputFunction), false, "String")] + [InlineData("StringArrayInputFunction", nameof(CardinalityMany.StringArrayInputFunction), true, "String")] + [InlineData("BinaryInputFunction", nameof(CardinalityMany.BinaryInputFunction), false, "Binary")] + [InlineData("BinaryArrayInputFunction", nameof(CardinalityMany.BinaryArrayInputFunction), true, "Binary")] + [InlineData("IntArrayInputFunction", nameof(CardinalityMany.IntArrayInputFunction), true, "")] + [InlineData("StringListInputFunction", nameof(CardinalityMany.StringListInputFunction), true, "String")] + [InlineData("BinaryListInputFunction", nameof(CardinalityMany.BinaryListInputFunction), true, "Binary")] + [InlineData("EnumerableNestedStringGenericClassInputFunction", nameof(CardinalityMany.EnumerableNestedStringGenericClassInputFunction), true, "String")] + [InlineData("EnumerableNestedStringGenericClass2InputFunction", nameof(CardinalityMany.EnumerableNestedStringGenericClass2InputFunction), true, "String")] + [InlineData("IntListInputFunction", nameof(CardinalityMany.IntListInputFunction), true, "")] + [InlineData("StringDoubleArrayInputFunction", nameof(CardinalityMany.StringDoubleArrayInputFunction), true, "")] + [InlineData("EnumerableClassInputFunction", nameof(CardinalityMany.EnumerableClassInputFunction), true, "")] + [InlineData("EnumerableStringClassInputFunction", nameof(CardinalityMany.EnumerableStringClassInputFunction), true, "String")] + [InlineData("EnumerableBinaryClassInputFunction", nameof(CardinalityMany.EnumerableBinaryClassInputFunction), true, "Binary")] + [InlineData("EnumerableGenericClassInputFunction", nameof(CardinalityMany.EnumerableGenericClassInputFunction), true, "")] + [InlineData("EnumerableNestedBinaryClassInputFunction", nameof(CardinalityMany.EnumerableNestedBinaryClassInputFunction), true, "Binary")] + [InlineData("EnumerableNestedStringClassInputFunction", nameof(CardinalityMany.EnumerableNestedStringClassInputFunction), true, "String")] + [InlineData("LookupInputFunction", nameof(CardinalityMany.LookupInputFunction), false, "")] + [InlineData("DictionaryInputFunction", nameof(CardinalityMany.DictionaryInputFunction), false, "")] + [InlineData("ConcurrentDictionaryInputFunction", nameof(CardinalityMany.ConcurrentDictionaryInputFunction), false, "")] + [InlineData("HashSetInputFunction", nameof(CardinalityMany.HashSetInputFunction), true, "String")] + [InlineData("EnumerableInputFunction", nameof(CardinalityMany.EnumerableInputFunction), true, "")] + [InlineData("EnumerableStringInputFunction", nameof(CardinalityMany.EnumerableStringInputFunction), true, "String")] + [InlineData("EnumerableBinaryInputFunction", nameof(CardinalityMany.EnumerableBinaryInputFunction), true, "Binary")] + [InlineData("EnumerableGenericInputFunction", nameof(CardinalityMany.EnumerableGenericInputFunction), true, "")] + [InlineData("EnumerablePocoInputFunction", nameof(CardinalityMany.EnumerablePocoInputFunction), true, "")] + [InlineData("ListPocoInputFunction", nameof(CardinalityMany.ListPocoInputFunction), true, "")] + public void CardinalityManyFunctions(string functionName, string entryPoint, bool cardinalityMany, string dataType) + { + var generator = new FunctionMetadataGenerator(); + var typeDef = TestUtility.GetTypeDefinition(typeof(CardinalityMany)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + SdkFunctionMetadata metadata = functions.Where(a => string.Equals(a.Name, functionName, StringComparison.Ordinal)).Single(); + + ValidateFunction(metadata, functionName, GetEntryPoint(nameof(CardinalityMany), entryPoint), + b => ValidateTrigger(b, cardinalityMany)); + + AssertDictionary(extensions, new Dictionary(){ + { "Microsoft.Azure.WebJobs.Extensions.EventHubs", "4.2.0" } + }); + + void ValidateTrigger(ExpandoObject b, bool many) + { + var expected = new Dictionary() + { + { "Name", "input" }, + { "Type", "EventHubTrigger" }, + { "Direction", "In" }, + { "eventHubName", "test" }, + { "Connection", "EventHubConnectionAppSetting" } + }; + + if (many) + { + expected.Add("Cardinality", "Many"); + } + + if (!string.IsNullOrEmpty(dataType)) + { + expected.Add("DataType", dataType); + } + + AssertExpandoObject(b, expected); + } + } + private static string GetEntryPoint(string className, string methodName) => $"{typeof(FunctionMetadataGeneratorTests).FullName}+{className}.{methodName}"; private void ValidateFunction(SdkFunctionMetadata sdkFunctionMetadata, string name, string entryPoint, params Action[] bindingValidations) @@ -429,7 +505,7 @@ private static void AssertExpandoObject(ExpandoObject expando, IDictionary(IDictionary dict, IDictionary expected) + private static void AssertDictionary(IDictionary dict, IDictionary expected) { Assert.Equal(expected.Count, dict.Count); @@ -568,5 +644,270 @@ public Task RunTimer([TimerTrigger("0 0 0 * * *", RunOnStartup = false)] object throw new NotImplementedException(); } } + + private class CardinalityMany + { + [Function("StringInputFunction")] + public static void StringInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("StringArrayInputFunction")] + public static void StringArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("BinaryInputFunction")] + public static void BinaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] byte[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("BinaryArrayInputFunction")] + public static void BinaryArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] byte[][] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("IntArrayInputFunction")] + public static void IntArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] int[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("StringListInputFunction")] + public static void StringListInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("StringDoubleArrayInputFunction")] + public static void StringDoubleArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string[][] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("BinaryListInputFunction")] + public static void BinaryListInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("IntListInputFunction")] + public static void IntListInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] int[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableClassInputFunction")] + public static void EnumerableClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableStringClassInputFunction")] + public static void EnumerableStringClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableBinaryClassInputFunction")] + public static void EnumerableBinaryClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableBinaryTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableGenericClassInputFunction")] + public static void EnumerableGenericClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableGenericTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedBinaryClassInputFunction")] + public static void EnumerableNestedBinaryClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableBinaryNestedTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedStringClassInputFunction")] + public static void EnumerableNestedStringClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringNestedTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedStringGenericClassInputFunction")] + public static void EnumerableNestedStringGenericClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringNestedGenericTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedStringGenericClass2InputFunction")] + public static void EnumerableNestedStringGenericClass2InputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringNestedGenericTestClass2 input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("LookupInputFunction")] + public static void LookupInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] Lookup input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("DictionaryInputFunction")] + public static void DictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] Dictionary input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("ConcurrentDictionaryInputFunction")] + public static void ConcurrentDictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] ConcurrentDictionary input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("HashSetInputFunction")] + public static void HashSetInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] HashSet input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableInputFunction")] + public static void EnumerableInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableStringInputFunction")] + public static void EnumerableStringInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableBinaryInputFunction")] + public static void EnumerableBinaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableGenericInputFunction")] + public static void EnumerableGenericInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerablePocoInputFunction")] + public static void EnumerablePocoInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("ListPocoInputFunction")] + public static void ListPocoInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + throw new NotImplementedException(); + } + } + + private class EnumerableTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class EnumerableStringTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class EnumerableBinaryTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class EnumerableStringTestClass : List + { + } + + private class EnumerableBinaryNestedTestClass : EnumerableBinaryTestClass + { + } + + private class EnumerableStringNestedTestClass : EnumerableStringTestClass + { + } + + private class EnumerableStringNestedGenericTestClass2 : EnumerableStringNestedGenericTestClass + { + } + + private class EnumerableStringNestedGenericTestClass : EnumerableStringTestClass + { + } + + private class EnumerableGenericTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class Poco + { + public string Foo; + public string Bar; + } } } diff --git a/test/FunctionMetadataGeneratorTests/SdkTests.csproj b/test/FunctionMetadataGeneratorTests/SdkTests.csproj index 34ab4a2f1..2275a2f5f 100644 --- a/test/FunctionMetadataGeneratorTests/SdkTests.csproj +++ b/test/FunctionMetadataGeneratorTests/SdkTests.csproj @@ -28,6 +28,7 @@ + From b88044bc171e420f5b3e50fcec94403b1b9e1834 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Tue, 2 Mar 2021 13:10:55 -0800 Subject: [PATCH 02/24] update sdk e2e test for datatype --- test/SdkE2ETests/Contents/functions.metadata | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/SdkE2ETests/Contents/functions.metadata b/test/SdkE2ETests/Contents/functions.metadata index 7758da8f6..4f450d0d0 100644 --- a/test/SdkE2ETests/Contents/functions.metadata +++ b/test/SdkE2ETests/Contents/functions.metadata @@ -22,6 +22,7 @@ "name": "myBlob", "type": "Blob", "direction": "In", + "dataType": "String", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage" }, @@ -59,6 +60,7 @@ "name": "myBlob", "type": "Blob", "direction": "In", + "dataType": "String", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage" } From 992201adf08dd9605c6f024e4dd65137981a9505 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Tue, 2 Mar 2021 20:43:32 -0800 Subject: [PATCH 03/24] move cardinality and datatype stuff to later --- sdk/Sdk/FunctionMetadataGenerator.cs | 57 ++++++++----------- .../FunctionMetadataGeneratorTests.cs | 5 +- test/SdkE2ETests/Contents/functions.metadata | 1 + 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 84cbdb9c0..d4cc7b504 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -292,7 +292,7 @@ private void AddOutputBindingFromProperty(IList bindingMetadata, foundOutputAttribute = true; - AddOutputBindingMetadata(bindingMetadata, propertyAttribute, property.Name); + AddOutputBindingMetadata(bindingMetadata, propertyAttribute, property.PropertyType, property.Name); AddExtensionInfo(_extensions, propertyAttribute); } } @@ -312,7 +312,7 @@ private bool TryAddOutputBindingFromMethod(IList bindingMetadata, $"Please use an encapsulation to define the bindings in properties. For more information: https://aka.ms/dotnet-worker-poco-binding."); } - AddOutputBindingMetadata(bindingMetadata, methodAttribute, ReturnBindingName); + AddOutputBindingMetadata(bindingMetadata, methodAttribute, methodAttribute.AttributeType, ReturnBindingName); AddExtensionInfo(_extensions, methodAttribute); foundBinding = true; @@ -330,7 +330,7 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe { if (IsFunctionBindingType(parameterAttribute)) { - AddBindingMetadata(bindingMetadata, parameterAttribute, parameter.Name, parameter.ParameterType); + AddBindingMetadata(bindingMetadata, parameterAttribute, parameter.ParameterType, parameter.Name); AddExtensionInfo(_extensions, parameterAttribute); } } @@ -357,20 +357,20 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe } } - private static void AddOutputBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? name = null) + private static void AddOutputBindingMetadata(IList bindingMetadata, CustomAttribute attribute, TypeReference parameterType, string? name = null) { - AddBindingMetadata(bindingMetadata, attribute, parameterName: name, parameterType: null); + AddBindingMetadata(bindingMetadata, attribute, parameterType, parameterName: name); } - private static void AddBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? parameterName, TypeReference? parameterType) + private static void AddBindingMetadata(IList bindingMetadata, CustomAttribute attribute, TypeReference parameterType, string? parameterName) { string bindingType = GetBindingType(attribute); - ExpandoObject binding = BuildBindingMetadataFromAttribute(attribute, bindingType, parameterName, parameterType); + ExpandoObject binding = BuildBindingMetadataFromAttribute(attribute, bindingType, parameterType, parameterName); bindingMetadata.Add(binding); } - private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute attribute, string bindingType, string? parameterName, TypeReference? parameterType) + private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute attribute, string bindingType, TypeReference parameterType, string? parameterName) { ExpandoObject binding = new ExpandoObject(); @@ -384,38 +384,29 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict["Type"] = bindingType; bindingDict["Direction"] = GetBindingDirection(attribute); - // Inspect parameter type - if (parameterType is not null) + // Is string parameter type + if (IsStringType(parameterType.FullName)) { - // Is string parameter type - if (IsStringType(parameterType.FullName)) + bindingDict["DataType"] = "String"; + } + // Is binary parameter type + else if (IsBinaryType(parameterType.FullName)) + { + bindingDict["DataType"] = "Binary"; + } + + // Add "cardinality": "many" if we see an IEnumerable type or array type + if (IsIterableCollection(parameterType, out DataType dataType)) + { + bindingDict["Cardinality"] = "Many"; + if (dataType.Equals(DataType.String)) { bindingDict["DataType"] = "String"; } - - // Is binary parameter type - if (IsBinaryType(parameterType.FullName)) + else if (dataType.Equals(DataType.Binary)) { bindingDict["DataType"] = "Binary"; } - - // Trigger logic - if (bindingType.EndsWith("Trigger", StringComparison.OrdinalIgnoreCase)) - { - // Add "cardinality": "many" if we see an IEnumerable type or array type - if (IsIterableCollection(parameterType, out DataType dataType)) - { - bindingDict["Cardinality"] = "Many"; - if (dataType.Equals(DataType.String)) - { - bindingDict["DataType"] = "String"; - } - else if (dataType.Equals(DataType.Binary)) - { - bindingDict["DataType"] = "Binary"; - } - } - } } foreach (var property in attribute.GetAllDefinedProperties()) diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 70aa097f0..4f6442654 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -287,7 +287,8 @@ void ValidateBlobOutput(ExpandoObject b) { "Type", "Blob" }, { "Direction", "Out" }, { "blobPath", "container1/hello.txt" }, - { "Connection", "MyOtherConnection" } + { "Connection", "MyOtherConnection" }, + { "DataType", "String" } }); } @@ -299,6 +300,7 @@ void ValidateQueueOutput(ExpandoObject b) { "Type", "Queue" }, { "Direction", "Out" }, { "queueName", "queue2" }, + { "DataType", "String" } }); } } @@ -356,6 +358,7 @@ void ValidateQueueOutput(ExpandoObject b) { "Type", "Queue" }, { "Direction", "Out" }, { "queueName", "queue2" }, + { "DataType", "String" } }); } } diff --git a/test/SdkE2ETests/Contents/functions.metadata b/test/SdkE2ETests/Contents/functions.metadata index 4f450d0d0..963dd07ea 100644 --- a/test/SdkE2ETests/Contents/functions.metadata +++ b/test/SdkE2ETests/Contents/functions.metadata @@ -89,6 +89,7 @@ "name": "Name", "type": "Queue", "direction": "Out", + "dataType": "String", "queueName": "functionstesting2", "connection": "AzureWebJobsStorage" }, From 2d527aad46e2042b97de0e6d4fc64dff7ee22bf2 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Tue, 2 Mar 2021 20:53:26 -0800 Subject: [PATCH 04/24] resolve typeof's --- sdk/Sdk/FunctionMetadataGenerator.cs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index d4cc7b504..9e3f18c7b 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -24,11 +24,17 @@ internal class FunctionMetadataGenerator private const string VoidType = "System.Void"; private const string ReturnBindingName = "$return"; private const string HttpTriggerBindingType = "HttpTrigger"; + private const string IEnumerableType = "System.Collections.IEnumerable"; + private const string IEnumerableGenericType = "System.Collections.Generic.IEnumerable`1"; private const string IEnumerableOfStringType = "System.Collections.Generic.IEnumerable`1"; private const string IEnumerableOfBinaryType = "System.Collections.Generic.IEnumerable`1"; private const string IEnumerableOfT = "System.Collections.Generic.IEnumerable`1"; private const string IEnumerableOfKeyValuePair = "System.Collections.Generic.IEnumerable`1>"; private const string GenericIEnumerableArgumentName = "T"; + private const string StringType = "System.String"; + private const string ByteArrayType = "System.Byte[]"; + private const string LookupGenericType = "System.Linq.Lookup`2"; + private const string DictionaryGenericType = "System.Collections.Generic.Dictionary`2"; private readonly IndentableLogger _logger; @@ -420,7 +426,7 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a private static bool IsIterableCollection(TypeReference type, out DataType dataType) { // Array and not byte array - bool isArray = type.IsArray && !string.Equals(type.FullName, typeof(byte[]).FullName, StringComparison.Ordinal); + bool isArray = type.IsArray && !string.Equals(type.FullName, ByteArrayType, StringComparison.Ordinal); if (isArray) { TypeSpecification? typeSpecification = type as TypeSpecification; @@ -432,8 +438,8 @@ private static bool IsIterableCollection(TypeReference type, out DataType dataTy } bool isMappingEnumerable = IsOrDerivedFrom(type, IEnumerableOfKeyValuePair) - || IsOrDerivedFrom(type, typeof(Lookup<,>).FullName) - || IsOrDerivedFrom(type, typeof(Dictionary<,>).FullName); + || IsOrDerivedFrom(type, LookupGenericType) + || IsOrDerivedFrom(type, DictionaryGenericType); if (isMappingEnumerable) { dataType = DataType.Undefined; @@ -444,8 +450,8 @@ private static bool IsIterableCollection(TypeReference type, out DataType dataTy bool isEnumerableOfT = IsOrDerivedFrom(type, IEnumerableOfT); bool isEnumerableCollection = !IsStringType(type.FullName) - && (IsOrDerivedFrom(type, typeof(IEnumerable).FullName) - || IsOrDerivedFrom(type, typeof(IEnumerable<>).FullName) + && (IsOrDerivedFrom(type, IEnumerableType) + || IsOrDerivedFrom(type, IEnumerableGenericType) || isEnumerableOfT); if (isEnumerableCollection) { @@ -566,12 +572,12 @@ private static DataType GetDataTypeFromType(string fullName) private static bool IsStringType(string fullName) { - return string.Equals(fullName, typeof(string).FullName, StringComparison.Ordinal); + return string.Equals(fullName, StringType, StringComparison.Ordinal); } private static bool IsBinaryType(string fullName) { - return string.Equals(fullName, typeof(byte[]).FullName, StringComparison.Ordinal); + return string.Equals(fullName, ByteArrayType, StringComparison.Ordinal); } private static string GetBindingType(CustomAttribute attribute) From dbfd4471c5803178ce0d4972c4103f8c411d3f01 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Wed, 3 Mar 2021 20:08:48 -0800 Subject: [PATCH 05/24] cardinality through IBatchedInput - tests need changes --- .../IBatchedInput.cs | 11 +++++ .../InputBindingAttribute.cs | 2 +- .../EventHubTriggerAttribute.cs | 10 ++++- .../KafkaTriggerAttribute.cs | 6 ++- .../ServiceBusTriggerAttribute.cs | 7 ++- sdk/Sdk/FunctionMetadataGenerator.cs | 43 +++++++++++++----- .../FunctionMetadataGeneratorTests.cs | 44 +++++++++++++++++++ 7 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 extensions/Worker.Extensions.Abstractions/IBatchedInput.cs diff --git a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs new file mode 100644 index 000000000..62fefadeb --- /dev/null +++ b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions +{ public interface IBatchedInput + { + public bool IsBatched { get; set; } + } +} diff --git a/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs b/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs index d95cee778..7f7507509 100644 --- a/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs +++ b/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System; diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index f5d0ac59d..e2f798a6a 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -5,15 +5,16 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class EventHubTriggerAttribute : TriggerBindingAttribute + public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, IBatchedInput { /// /// Create an instance of this attribute. /// /// Event hub to listen on for messages. - public EventHubTriggerAttribute(string eventHubName) + public EventHubTriggerAttribute(string eventHubName, bool isBatched = true) { EventHubName = eventHubName; + IsBatched = isBatched; } /// @@ -30,5 +31,10 @@ public EventHubTriggerAttribute(string eventHubName) /// Gets or sets the optional app setting name that contains the Event Hub connection string. If missing, tries to use a registered event hub receiver. /// public string? Connection { get; set; } + + /// + /// Configures trigger to process events in batches or one at a time. Default value is "true". + /// + public bool IsBatched { get; set; } } } diff --git a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs index e75bc69b3..f39a9dd8f 100644 --- a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs @@ -8,7 +8,7 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class KafkaTriggerAttribute : TriggerBindingAttribute + public sealed class KafkaTriggerAttribute : TriggerBindingAttribute, IBatchedInput { public KafkaTriggerAttribute(string brokerList, string topic) { @@ -95,5 +95,9 @@ public KafkaTriggerAttribute(string brokerList, string topic) /// public string? SslKeyPassword { get; set; } + /// + /// Configures trigger to process events in batches or one at a time. Default value is "false". + /// + public bool IsBatched { get; set; } } } diff --git a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs index 3e0580e6a..fa6cffe7a 100644 --- a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs +++ b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs @@ -5,7 +5,7 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class ServiceBusTriggerAttribute : TriggerBindingAttribute + public sealed class ServiceBusTriggerAttribute : TriggerBindingAttribute, IBatchedInput { private readonly string? _queueName; private readonly string? _topicName; @@ -67,5 +67,10 @@ public string? SubscriptionName /// Gets or sets a value indicating whether the sessions are enabled. /// public bool IsSessionsEnabled { get; set; } + + /// + /// Configures trigger to process events in batches or one at a time. Default value is "false". + /// + public bool IsBatched { get; set; } } } diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 9e3f18c7b..26465a26d 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -401,23 +401,44 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict["DataType"] = "Binary"; } - // Add "cardinality": "many" if we see an IEnumerable type or array type - if (IsIterableCollection(parameterType, out DataType dataType)) + foreach (var property in attribute.GetAllDefinedProperties()) + { + bindingDict.Add(property.Key, property.Value); + } + + // TODO: do not rely on property alone + if (bindingDict["isBatched"] is not null + && bindingDict["isBatched"] is bool isBatchedValue) { - bindingDict["Cardinality"] = "Many"; - if (dataType.Equals(DataType.String)) + // Batching set to true + if (isBatchedValue) { - bindingDict["DataType"] = "String"; + bindingDict["Cardinality"] = "Many"; + // Throw if parameter type is not IEnumerable + if (IsIterableCollection(parameterType, out DataType dataType)) + { + if (dataType.Equals(DataType.String)) + { + bindingDict["DataType"] = "String"; + } + else if (dataType.Equals(DataType.Binary)) + { + bindingDict["DataType"] = "Binary"; + } + } + else + { + throw new Exception("Function is configured to process events in batches but parameter type is not iterable. " + + $"Change parameter { parameterName ?? "type" } to be an IEnumerable type or set 'IsBatched' to false on your {attribute.AttributeType.Name.Replace("Attribute", "")} attribute."); + } } - else if (dataType.Equals(DataType.Binary)) + // Batching set to false + else { - bindingDict["DataType"] = "Binary"; + bindingDict["Cardinality"] = "One"; } - } - foreach (var property in attribute.GetAllDefinedProperties()) - { - bindingDict.Add(property.Key, property.Value); + bindingDict.Remove("isBatched"); } return binding; diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 4f6442654..a01d305ee 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -486,6 +486,50 @@ void ValidateTrigger(ExpandoObject b, bool many) } } + [Fact] + public void CardinalityManyAttribute() + { + var generator = new FunctionMetadataGenerator(); + var typeDef = TestUtility.GetTypeDefinition(typeof(TestingClass)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + SdkFunctionMetadata metadata = functions.Where(a => string.Equals(a.Name, "EventHubTrigger", StringComparison.Ordinal)).Single(); + + ValidateFunction(metadata, "EventHubTrigger", GetEntryPoint(nameof(TestingClass), nameof(TestingClass.EventHubTrigger)), + b => ValidateTrigger(b)); + + AssertDictionary(extensions, new Dictionary(){ + { "Microsoft.Azure.WebJobs.Extensions.EventHubs", "4.2.0" } + }); + + void ValidateTrigger(ExpandoObject b) + { + var expected = new Dictionary() + { + { "Name", "input" }, + { "Type", "EventHubTrigger" }, + { "Direction", "In" }, + { "eventHubName", "test" }, + { "Connection", "EventHubConnectionAppSetting" }, + { "Cardinality", "Many" }, + { "DataType", "String" }, + }; + + AssertExpandoObject(b, expected); + } + } + + private class TestingClass + { + [Function("EventHubTrigger")] + public static void EventHubTrigger([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + } + private static string GetEntryPoint(string className, string methodName) => $"{typeof(FunctionMetadataGeneratorTests).FullName}+{className}.{methodName}"; private void ValidateFunction(SdkFunctionMetadata sdkFunctionMetadata, string name, string entryPoint, params Action[] bindingValidations) From c51705d054247d56c380ffc8331ddef29a79e22e Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Wed, 3 Mar 2021 20:12:40 -0800 Subject: [PATCH 06/24] re-add removed stuff --- .../Context/Features/GrpcFunctionBindingsFeature.cs | 6 ++++++ src/DotNetWorker/GrpcWorker.cs | 1 + 2 files changed, 7 insertions(+) diff --git a/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs b/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs index 837c92485..e30d9868b 100644 --- a/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs +++ b/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs @@ -116,6 +116,12 @@ public void SetOutputBinding(string name, object value) // This is guaranteed to be Json here -- we can use that. TypedData.DataOneofCase.Json => typedData.Json, TypedData.DataOneofCase.Bytes => typedData.Bytes.Memory, + TypedData.DataOneofCase.CollectionBytes => typedData.CollectionBytes.Bytes.Select(element => { + return element.Memory.ToArray(); + }), + TypedData.DataOneofCase.CollectionString => typedData.CollectionString.String, + TypedData.DataOneofCase.CollectionDouble => typedData.CollectionDouble.Double, + TypedData.DataOneofCase.CollectionSint64 => typedData.CollectionSint64.Sint64, _ => throw new NotSupportedException($"{typedData.DataCase} is not supported."), }; } diff --git a/src/DotNetWorker/GrpcWorker.cs b/src/DotNetWorker/GrpcWorker.cs index eb30b1bb9..fe2bcc886 100644 --- a/src/DotNetWorker/GrpcWorker.cs +++ b/src/DotNetWorker/GrpcWorker.cs @@ -207,6 +207,7 @@ internal async Task WorkerInitRequestHandlerAsync(StreamingMessage request) response.Capabilities.Add("RawHttpBodyBytes", bool.TrueString); response.Capabilities.Add("RpcHttpTriggerMetadataRemoved", bool.TrueString); response.Capabilities.Add("UseNullableValueDictionaryForHttp", bool.TrueString); + response.Capabilities.Add("TypedDataCollection", bool.TrueString); StreamingMessage responseMessage = new StreamingMessage { From e97c8835e70bdf767d41ea539339999c037bcd95 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Wed, 3 Mar 2021 20:18:14 -0800 Subject: [PATCH 07/24] fix spacing --- extensions/Worker.Extensions.Abstractions/IBatchedInput.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs index 62fefadeb..31bf61d0f 100644 --- a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs +++ b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs @@ -4,7 +4,8 @@ using System; namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions -{ public interface IBatchedInput +{ + public interface IBatchedInput { public bool IsBatched { get; set; } } From c178301c2be89cea4d72b401224e53befe3c9092 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Wed, 3 Mar 2021 21:59:44 -0800 Subject: [PATCH 08/24] add comments --- .../Worker.Extensions.Abstractions/IBatchedInput.cs | 10 ++++++++++ sdk/Sdk/FunctionMetadataGenerator.cs | 11 ++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs index 31bf61d0f..cbff0f94e 100644 --- a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs +++ b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs @@ -7,6 +7,16 @@ namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions { public interface IBatchedInput { + /// + /// Configures trigger to process events in batches or one at a time. + /// This translates to values for the "cardinality" property in WebJobs terms. + /// true => "Many" + /// false => "One" + /// + /// To default to a particular true or false, the constructor of the attribute that inherits + /// from this must include 'bool isBatched = [true / false]' as an optional parameters + /// on the constructor + /// public bool IsBatched { get; set; } } } diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 26465a26d..453d26ccb 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Dynamic; @@ -406,12 +405,18 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict.Add(property.Key, property.Value); } + // Determine if we should set the "Cardinality" property based on + // the presence of "isBatched." This is a property that is from the IBatchedInput + // interface. + // Conversion rule + // "isBatched": true => "Cardinality": "Many" + // "isBatched": false => "Cardinality": "One" // TODO: do not rely on property alone if (bindingDict["isBatched"] is not null - && bindingDict["isBatched"] is bool isBatchedValue) + && bindingDict["isBatched"] is bool isBatched) { // Batching set to true - if (isBatchedValue) + if (isBatched) { bindingDict["Cardinality"] = "Many"; // Throw if parameter type is not IEnumerable From e9d95d2e5b47d0507f4f88408d4b5c7e0733faaf Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Wed, 3 Mar 2021 23:16:40 -0800 Subject: [PATCH 09/24] Update tests and make SDK generation exceptions surface as build error --- .../EventHubTriggerAttribute.cs | 4 +- sdk/Sdk/CustomAttributeExtensions.cs | 13 +++-- sdk/Sdk/FunctionMetadataGenerator.cs | 29 ++++++---- .../FunctionsMetadataGenerationException.cs | 12 ++++ .../EventHubs/EventHubsObjectFunctions.cs | 4 +- .../FunctionMetadataGeneratorTests.cs | 56 ++++++------------- 6 files changed, 59 insertions(+), 59 deletions(-) create mode 100644 sdk/Sdk/FunctionsMetadataGenerationException.cs diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index e2f798a6a..3fb4d2df7 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -11,10 +11,10 @@ public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, IBatched /// Create an instance of this attribute. /// /// Event hub to listen on for messages. - public EventHubTriggerAttribute(string eventHubName, bool isBatched = true) + public EventHubTriggerAttribute(string eventHubName, bool IsBatched = true) { EventHubName = eventHubName; - IsBatched = isBatched; + this.IsBatched = IsBatched; } /// diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index a1644a8fd..535911f8b 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System; @@ -34,14 +34,14 @@ private static void LoadConstructorArguments(IDictionary propert string? paramName = param?.Name; object? paramValue = arg.Value; - if (paramName == null || paramValue == null) + if (paramName is null || paramValue is null) { continue; } paramValue = GetEnrichedValue(param!.ParameterType, paramValue); - - properties[paramName] = paramValue!; + + properties[paramName.ToLower()] = paramValue!; } } @@ -50,15 +50,16 @@ private static void LoadDefinedProperties(IDictionary properties foreach (CustomAttributeNamedArgument property in attribute.Properties) { object? propVal = property.Argument.Value; + string? propName = property.Name; - if (propVal == null) + if (propVal is null || propName is null) { continue; } propVal = GetEnrichedValue(property.Argument.Type, propVal); - properties[property.Name] = propVal!; + properties[propName.ToLower()] = propVal!; } } diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 453d26ccb..92c0ec8b9 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -34,6 +34,7 @@ internal class FunctionMetadataGenerator private const string ByteArrayType = "System.Byte[]"; private const string LookupGenericType = "System.Linq.Lookup`2"; private const string DictionaryGenericType = "System.Collections.Generic.Dictionary`2"; + private const string IsBatchedKey = "isbatched"; private readonly IndentableLogger _logger; @@ -101,6 +102,11 @@ public IEnumerable GenerateFunctionMetadata(string assembly { _logger.LogMessage($"Skipping file '{Path.GetFileName(path)}' because of a {nameof(BadImageFormatException)}."); } + catch (FunctionsMetadataGenerationException ex) + { + _logger.LogError($"Failed to generate function metadata from {Path.GetFileName(path)}."); + throw ex; + } catch (Exception ex) { _logger.LogWarning($"Could not evaluate '{Path.GetFileName(path)}' for functions metadata. Exception message: {ex.ToString()}"); @@ -234,7 +240,7 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat else { TypeDefinition returnDefinition = returnType.Resolve() - ?? throw new InvalidOperationException($"Couldn't find the type definition {returnType}"); + ?? throw new FunctionsMetadataGenerationException($"Couldn't find the type definition {returnType}"); bool hasOutputModel = TryAddOutputBindingsFromProperties(bindingMetadata, returnDefinition); @@ -266,7 +272,7 @@ private bool TryAddOutputBindingsFromProperties(IList bindingMeta { if (foundHttpOutput) { - throw new InvalidOperationException($"Found multiple public properties with type '{HttpResponseType}' defined in output type '{typeDefinition.FullName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple public properties with type '{HttpResponseType}' defined in output type '{typeDefinition.FullName}'. " + $"Only one HTTP response binding type is supported in your return type definition."); } @@ -291,7 +297,7 @@ private void AddOutputBindingFromProperty(IList bindingMetadata, { if (foundOutputAttribute) { - throw new InvalidOperationException($"Found multiple output attributes on property '{property.Name}' defined in the function return type '{typeName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple output attributes on property '{property.Name}' defined in the function return type '{typeName}'. " + $"Only one output binding attribute is is supported on a property."); } @@ -313,7 +319,7 @@ private bool TryAddOutputBindingFromMethod(IList bindingMetadata, { if (foundBinding) { - throw new Exception($"Found multiple Output bindings on method '{method.FullName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple Output bindings on method '{method.FullName}'. " + $"Please use an encapsulation to define the bindings in properties. For more information: https://aka.ms/dotnet-worker-poco-binding."); } @@ -409,11 +415,12 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a // the presence of "isBatched." This is a property that is from the IBatchedInput // interface. // Conversion rule - // "isBatched": true => "Cardinality": "Many" - // "isBatched": false => "Cardinality": "One" + // "isbatched": true => "Cardinality": "Many" + // "isbatched": false => "Cardinality": "One" // TODO: do not rely on property alone - if (bindingDict["isBatched"] is not null - && bindingDict["isBatched"] is bool isBatched) + + if (bindingDict.TryGetValue(IsBatchedKey, out object isBatchedValue) + && isBatchedValue is bool isBatched) { // Batching set to true if (isBatched) @@ -433,8 +440,8 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a } else { - throw new Exception("Function is configured to process events in batches but parameter type is not iterable. " + - $"Change parameter { parameterName ?? "type" } to be an IEnumerable type or set 'IsBatched' to false on your {attribute.AttributeType.Name.Replace("Attribute", "")} attribute."); + throw new FunctionsMetadataGenerationException("Function is configured to process events in batches but parameter type is not iterable. " + + $"Change parameter { "'" + parameterName + "'" ?? "type" } to be an IEnumerable type or set 'IsBatched' to false on your '{attribute.AttributeType.Name.Replace("Attribute", "")}' attribute."); } } // Batching set to false @@ -443,7 +450,7 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict["Cardinality"] = "One"; } - bindingDict.Remove("isBatched"); + bindingDict.Remove(IsBatchedKey); } return binding; diff --git a/sdk/Sdk/FunctionsMetadataGenerationException.cs b/sdk/Sdk/FunctionsMetadataGenerationException.cs new file mode 100644 index 000000000..1aa12f862 --- /dev/null +++ b/sdk/Sdk/FunctionsMetadataGenerationException.cs @@ -0,0 +1,12 @@ +using System; + +namespace Microsoft.Azure.Functions.Worker.Sdk +{ + internal class FunctionsMetadataGenerationException: Exception + { + internal FunctionsMetadataGenerationException(string message): base(message) { } + + internal FunctionsMetadataGenerationException(string message, Exception innerException) : base(message, innerException) { } + + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs index d288a4b5b..44850db93 100644 --- a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs @@ -8,7 +8,7 @@ public static class EventHubsObjectFunctions { [Function(nameof(EventHubsObjectFunction))] [EventHubOutput("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] - public static TestData EventHubsObjectFunction([EventHubTrigger("test-eventhub-input-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] TestData input, + public static TestData EventHubsObjectFunction([EventHubTrigger("test-eventhub-input-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting", IsBatched = false)] TestData input, FunctionContext context) { var logger = context.GetLogger(nameof(EventHubsObjectFunction)); @@ -18,7 +18,7 @@ public static TestData EventHubsObjectFunction([EventHubTrigger("test-eventhub-i [Function(nameof(EventHubsVerifyOutputObject))] [QueueOutput("test-eventhub-output-object-dotnet-isolated")] - public static TestData EventHubsVerifyOutputObject([EventHubTrigger("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] TestData input, + public static TestData EventHubsVerifyOutputObject([EventHubTrigger("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting", IsBatched = false)] TestData input, FunctionContext context) { var logger = context.GetLogger(nameof(EventHubsVerifyOutputObject)); diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index a01d305ee..a4494479b 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -468,14 +468,18 @@ void ValidateTrigger(ExpandoObject b, bool many) { "Name", "input" }, { "Type", "EventHubTrigger" }, { "Direction", "In" }, - { "eventHubName", "test" }, - { "Connection", "EventHubConnectionAppSetting" } + { "eventhubname", "test" }, + { "connection", "EventHubConnectionAppSetting" } }; if (many) { expected.Add("Cardinality", "Many"); } + else + { + expected.Add("Cardinality", "One"); + } if (!string.IsNullOrEmpty(dataType)) { @@ -487,43 +491,19 @@ void ValidateTrigger(ExpandoObject b, bool many) } [Fact] - public void CardinalityManyAttribute() - { + public void CardinalityMany_WithNotIterableTypeThrows() + { var generator = new FunctionMetadataGenerator(); - var typeDef = TestUtility.GetTypeDefinition(typeof(TestingClass)); - var functions = generator.GenerateFunctionMetadata(typeDef); - var extensions = generator.Extensions; - - SdkFunctionMetadata metadata = functions.Where(a => string.Equals(a.Name, "EventHubTrigger", StringComparison.Ordinal)).Single(); - - ValidateFunction(metadata, "EventHubTrigger", GetEntryPoint(nameof(TestingClass), nameof(TestingClass.EventHubTrigger)), - b => ValidateTrigger(b)); - - AssertDictionary(extensions, new Dictionary(){ - { "Microsoft.Azure.WebJobs.Extensions.EventHubs", "4.2.0" } - }); + var typeDef = TestUtility.GetTypeDefinition(typeof(EventHubNotBatched)); - void ValidateTrigger(ExpandoObject b) - { - var expected = new Dictionary() - { - { "Name", "input" }, - { "Type", "EventHubTrigger" }, - { "Direction", "In" }, - { "eventHubName", "test" }, - { "Connection", "EventHubConnectionAppSetting" }, - { "Cardinality", "Many" }, - { "DataType", "String" }, - }; - - AssertExpandoObject(b, expected); - } + var exception = Assert.Throws(() => generator.GenerateFunctionMetadata(typeDef)); + Assert.Contains("Function is configured to process events in batches but parameter type is not iterable", exception.Message); } - private class TestingClass + private class EventHubNotBatched { [Function("EventHubTrigger")] - public static void EventHubTrigger([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string[] input, + public static void EventHubTrigger([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string input, FunctionContext context) { throw new NotImplementedException(); @@ -695,7 +675,7 @@ public Task RunTimer([TimerTrigger("0 0 0 * * *", RunOnStartup = false)] object private class CardinalityMany { [Function("StringInputFunction")] - public static void StringInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string input, + public static void StringInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] string input, FunctionContext context) { throw new NotImplementedException(); @@ -709,7 +689,7 @@ public static void StringArrayInputFunction([EventHubTrigger("test", Connection } [Function("BinaryInputFunction")] - public static void BinaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] byte[] input, + public static void BinaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] byte[] input, FunctionContext context) { throw new NotImplementedException(); @@ -814,21 +794,21 @@ public static void EnumerableNestedStringGenericClass2InputFunction([EventHubTri } [Function("LookupInputFunction")] - public static void LookupInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] Lookup input, + public static void LookupInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] Lookup input, FunctionContext context) { throw new NotImplementedException(); } [Function("DictionaryInputFunction")] - public static void DictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] Dictionary input, + public static void DictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] Dictionary input, FunctionContext context) { throw new NotImplementedException(); } [Function("ConcurrentDictionaryInputFunction")] - public static void ConcurrentDictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] ConcurrentDictionary input, + public static void ConcurrentDictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] ConcurrentDictionary input, FunctionContext context) { throw new NotImplementedException(); From 6815331d5abea10060a0e92e6bc4649270e6ec72 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 00:32:23 -0800 Subject: [PATCH 10/24] fixing low hanging cr changes --- .../EventHubTriggerAttribute.cs | 2 +- .../KafkaTriggerAttribute.cs | 2 +- .../ServiceBusTriggerAttribute.cs | 2 +- sdk/Sdk/Constants.cs | 34 ++++++++ sdk/Sdk/DataTypeEnum.cs | 5 +- sdk/Sdk/FunctionMetadataGenerator.cs | 81 +++++++------------ .../Converters/EnumerableConverter.cs | 56 ++++++------- 7 files changed, 92 insertions(+), 90 deletions(-) create mode 100644 sdk/Sdk/Constants.cs diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index 3fb4d2df7..a51e05c97 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -33,7 +33,7 @@ public EventHubTriggerAttribute(string eventHubName, bool IsBatched = true) public string? Connection { get; set; } /// - /// Configures trigger to process events in batches or one at a time. Default value is "true". + /// Gets or sets the configuration to enable batch processing of events. Default value is "true". /// public bool IsBatched { get; set; } } diff --git a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs index f39a9dd8f..ab36b0dd1 100644 --- a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs @@ -96,7 +96,7 @@ public KafkaTriggerAttribute(string brokerList, string topic) public string? SslKeyPassword { get; set; } /// - /// Configures trigger to process events in batches or one at a time. Default value is "false". + /// Gets or sets the configuration to enable batch processing of events. Default value is "true". /// public bool IsBatched { get; set; } } diff --git a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs index fa6cffe7a..1c54fb422 100644 --- a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs +++ b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs @@ -69,7 +69,7 @@ public string? SubscriptionName public bool IsSessionsEnabled { get; set; } /// - /// Configures trigger to process events in batches or one at a time. Default value is "false". + /// Gets or sets the configuration to enable batch processing of events. Default value is "true". /// public bool IsBatched { get; set; } } diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs new file mode 100644 index 000000000..a00156f83 --- /dev/null +++ b/sdk/Sdk/Constants.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.Sdk +{ + internal static class Constants + { + // Our types + internal const string BindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.BindingAttribute"; + internal const string OutputBindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.OutputBindingAttribute"; + internal const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; + internal const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; + internal const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; + + // System types + internal const string IEnumerableType = "System.Collections.IEnumerable"; + internal const string IEnumerableGenericType = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfStringType = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfBinaryType = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfT = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfKeyValuePair = "System.Collections.Generic.IEnumerable`1>"; + internal const string GenericIEnumerableArgumentName = "T"; + internal const string StringType = "System.String"; + internal const string ByteArrayType = "System.Byte[]"; + internal const string LookupGenericType = "System.Linq.Lookup`2"; + internal const string DictionaryGenericType = "System.Collections.Generic.Dictionary`2"; + internal const string TaskGenericType = "System.Threading.Tasks.Task`1"; + internal const string TaskType = "System.Threading.Tasks.Task"; + internal const string VoidType = "System.Void"; + + internal const string ReturnBindingName = "$return"; + internal const string HttpTriggerBindingType = "HttpTrigger"; + internal const string IsBatchedKey = "isbatched"; + } diff --git a/sdk/Sdk/DataTypeEnum.cs b/sdk/Sdk/DataTypeEnum.cs index 9a3ad85d2..4a8925f74 100644 --- a/sdk/Sdk/DataTypeEnum.cs +++ b/sdk/Sdk/DataTypeEnum.cs @@ -1,6 +1,5 @@ -using System; -using System.Collections.Generic; -using System.Text; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. namespace Microsoft.Azure.Functions.Worker.Sdk { diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 92c0ec8b9..2914e06c9 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -13,29 +13,6 @@ namespace Microsoft.Azure.Functions.Worker.Sdk { internal class FunctionMetadataGenerator { - private const string BindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.BindingAttribute"; - private const string OutputBindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.OutputBindingAttribute"; - private const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; - private const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; - private const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; - private const string TaskGenericType = "System.Threading.Tasks.Task`1"; - private const string TaskType = "System.Threading.Tasks.Task"; - private const string VoidType = "System.Void"; - private const string ReturnBindingName = "$return"; - private const string HttpTriggerBindingType = "HttpTrigger"; - private const string IEnumerableType = "System.Collections.IEnumerable"; - private const string IEnumerableGenericType = "System.Collections.Generic.IEnumerable`1"; - private const string IEnumerableOfStringType = "System.Collections.Generic.IEnumerable`1"; - private const string IEnumerableOfBinaryType = "System.Collections.Generic.IEnumerable`1"; - private const string IEnumerableOfT = "System.Collections.Generic.IEnumerable`1"; - private const string IEnumerableOfKeyValuePair = "System.Collections.Generic.IEnumerable`1>"; - private const string GenericIEnumerableArgumentName = "T"; - private const string StringType = "System.String"; - private const string ByteArrayType = "System.Byte[]"; - private const string LookupGenericType = "System.Linq.Lookup`2"; - private const string DictionaryGenericType = "System.Collections.Generic.Dictionary`2"; - private const string IsBatchedKey = "isbatched"; - private readonly IndentableLogger _logger; // TODO: Verify that we don't need to allow @@ -170,7 +147,7 @@ private bool TryCreateFunctionMetadata(MethodDefinition method, out SdkFunctionM foreach (CustomAttribute attribute in method.CustomAttributes) { - if (string.Equals(attribute.AttributeType.FullName, FunctionNameType, StringComparison.Ordinal)) + if (string.Equals(attribute.AttributeType.FullName, Constants.FunctionNameType, StringComparison.Ordinal)) { string functionName = attribute.ConstructorArguments.SingleOrDefault().Value.ToString(); @@ -231,11 +208,11 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat { TypeReference? returnType = GetTaskElementType(method.ReturnType); - if (returnType is not null && !string.Equals(returnType.FullName, VoidType, StringComparison.Ordinal)) + if (returnType is not null && !string.Equals(returnType.FullName, Constants.VoidType, StringComparison.Ordinal)) { - if (string.Equals(returnType.FullName, HttpResponseType, StringComparison.Ordinal)) + if (string.Equals(returnType.FullName, Constants.HttpResponseType, StringComparison.Ordinal)) { - AddHttpOutputBinding(bindingMetadata, ReturnBindingName); + AddHttpOutputBinding(bindingMetadata, Constants.ReturnBindingName); } else { @@ -249,7 +226,7 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat // support to other triggers without special handling if (!hasOutputModel && bindingMetadata.Any(d => IsHttpTrigger(d))) { - AddHttpOutputBinding(bindingMetadata, ReturnBindingName); + AddHttpOutputBinding(bindingMetadata, Constants.ReturnBindingName); } } } @@ -258,7 +235,7 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat private bool IsHttpTrigger(ExpandoObject bindingMetadata) { return bindingMetadata.Any(kvp => string.Equals(kvp.Key, "Type", StringComparison.Ordinal) - && string.Equals(kvp.Value?.ToString(), HttpTriggerBindingType, StringComparison.Ordinal)); + && string.Equals(kvp.Value?.ToString(), Constants.HttpTriggerBindingType, StringComparison.Ordinal)); } private bool TryAddOutputBindingsFromProperties(IList bindingMetadata, TypeDefinition typeDefinition) @@ -268,11 +245,11 @@ private bool TryAddOutputBindingsFromProperties(IList bindingMeta foreach (PropertyDefinition property in typeDefinition.Properties) { - if (string.Equals(property.PropertyType.FullName, HttpResponseType, StringComparison.Ordinal)) + if (string.Equals(property.PropertyType.FullName, Constants.HttpResponseType, StringComparison.Ordinal)) { if (foundHttpOutput) { - throw new FunctionsMetadataGenerationException($"Found multiple public properties with type '{HttpResponseType}' defined in output type '{typeDefinition.FullName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple public properties with type '{Constants.HttpResponseType}' defined in output type '{typeDefinition.FullName}'. " + $"Only one HTTP response binding type is supported in your return type definition."); } @@ -323,7 +300,7 @@ private bool TryAddOutputBindingFromMethod(IList bindingMetadata, $"Please use an encapsulation to define the bindings in properties. For more information: https://aka.ms/dotnet-worker-poco-binding."); } - AddOutputBindingMetadata(bindingMetadata, methodAttribute, methodAttribute.AttributeType, ReturnBindingName); + AddOutputBindingMetadata(bindingMetadata, methodAttribute, methodAttribute.AttributeType, Constants.ReturnBindingName); AddExtensionInfo(_extensions, methodAttribute); foundBinding = true; @@ -350,14 +327,14 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe private static TypeReference? GetTaskElementType(TypeReference typeReference) { - if (typeReference is null || typeReference.FullName == TaskType) + if (typeReference is null || typeReference.FullName == Constants.TaskType) { return null; } if (typeReference.IsGenericInstance && typeReference is GenericInstanceType genericType - && string.Equals(typeReference.GetElementType().FullName, TaskGenericType, StringComparison.Ordinal)) + && string.Equals(typeReference.GetElementType().FullName, Constants.TaskGenericType, StringComparison.Ordinal)) { // T from Task return genericType.GenericArguments[0]; @@ -419,7 +396,7 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a // "isbatched": false => "Cardinality": "One" // TODO: do not rely on property alone - if (bindingDict.TryGetValue(IsBatchedKey, out object isBatchedValue) + if (bindingDict.TryGetValue(Constants.IsBatchedKey, out object isBatchedValue) && isBatchedValue is bool isBatched) { // Batching set to true @@ -450,7 +427,7 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict["Cardinality"] = "One"; } - bindingDict.Remove(IsBatchedKey); + bindingDict.Remove(Constants.IsBatchedKey); } return binding; @@ -459,7 +436,7 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a private static bool IsIterableCollection(TypeReference type, out DataType dataType) { // Array and not byte array - bool isArray = type.IsArray && !string.Equals(type.FullName, ByteArrayType, StringComparison.Ordinal); + bool isArray = type.IsArray && !string.Equals(type.FullName, Constants.ByteArrayType, StringComparison.Ordinal); if (isArray) { TypeSpecification? typeSpecification = type as TypeSpecification; @@ -470,9 +447,9 @@ private static bool IsIterableCollection(TypeReference type, out DataType dataTy } } - bool isMappingEnumerable = IsOrDerivedFrom(type, IEnumerableOfKeyValuePair) - || IsOrDerivedFrom(type, LookupGenericType) - || IsOrDerivedFrom(type, DictionaryGenericType); + bool isMappingEnumerable = IsOrDerivedFrom(type, Constants.IEnumerableOfKeyValuePair) + || IsOrDerivedFrom(type, Constants.LookupGenericType) + || IsOrDerivedFrom(type, Constants.DictionaryGenericType); if (isMappingEnumerable) { dataType = DataType.Undefined; @@ -480,20 +457,20 @@ private static bool IsIterableCollection(TypeReference type, out DataType dataTy } // IEnumerable and not string or dictionary - bool isEnumerableOfT = IsOrDerivedFrom(type, IEnumerableOfT); + bool isEnumerableOfT = IsOrDerivedFrom(type, Constants.IEnumerableOfT); bool isEnumerableCollection = !IsStringType(type.FullName) - && (IsOrDerivedFrom(type, IEnumerableType) - || IsOrDerivedFrom(type, IEnumerableGenericType) + && (IsOrDerivedFrom(type, Constants.IEnumerableType) + || IsOrDerivedFrom(type, Constants.IEnumerableGenericType) || isEnumerableOfT); if (isEnumerableCollection) { dataType = DataType.Undefined; - if (IsOrDerivedFrom(type, IEnumerableOfStringType)) + if (IsOrDerivedFrom(type, Constants.IEnumerableOfStringType)) { dataType = DataType.String; } - else if (IsOrDerivedFrom(type, IEnumerableOfBinaryType)) + else if (IsOrDerivedFrom(type, Constants.IEnumerableOfBinaryType)) { dataType = DataType.Binary; } @@ -545,9 +522,9 @@ private static bool IsSubclassOf(TypeDefinition definition, string interfaceFull // Base case: // We are at IEnumerable and want to return the most recent resolution of T // (Most recent is relative to IEnumerable) - if (string.Equals(type.FullName, IEnumerableOfT, StringComparison.Ordinal)) + if (string.Equals(type.FullName, Constants.IEnumerableOfT, StringComparison.Ordinal)) { - if (foundMapping.TryGetValue(GenericIEnumerableArgumentName, out string typeName)) + if (foundMapping.TryGetValue(Constants.GenericIEnumerableArgumentName, out string typeName)) { return typeName; } @@ -605,12 +582,12 @@ private static DataType GetDataTypeFromType(string fullName) private static bool IsStringType(string fullName) { - return string.Equals(fullName, StringType, StringComparison.Ordinal); + return string.Equals(fullName, Constants.StringType, StringComparison.Ordinal); } private static bool IsBinaryType(string fullName) { - return string.Equals(fullName, ByteArrayType, StringComparison.Ordinal); + return string.Equals(fullName, Constants.ByteArrayType, StringComparison.Ordinal); } private static string GetBindingType(CustomAttribute attribute) @@ -640,7 +617,7 @@ private static void AddExtensionInfo(IDictionary extensions, Cus foreach (var assemblyAttribute in extensionAssemblyDefintion.CustomAttributes) { - if (string.Equals(assemblyAttribute.AttributeType.FullName, ExtensionsInformationType, StringComparison.Ordinal)) + if (string.Equals(assemblyAttribute.AttributeType.FullName, Constants.ExtensionsInformationType, StringComparison.Ordinal)) { string extensionName = assemblyAttribute.ConstructorArguments[0].Value.ToString(); string extensionVersion = assemblyAttribute.ConstructorArguments[1].Value.ToString(); @@ -665,12 +642,12 @@ private static string GetBindingDirection(CustomAttribute attribute) private static bool IsOutputBindingType(CustomAttribute attribute) { - return TryGetBaseAttributeType(attribute, OutputBindingType, out _); + return TryGetBaseAttributeType(attribute, Constants.OutputBindingType, out _); } private static bool IsFunctionBindingType(CustomAttribute attribute) { - return TryGetBaseAttributeType(attribute, BindingType, out _); + return TryGetBaseAttributeType(attribute, Constants.BindingType, out _); } private static bool TryGetBaseAttributeType(CustomAttribute attribute, string baseType, out TypeReference? baseTypeRef) diff --git a/src/DotNetWorker/Converters/EnumerableConverter.cs b/src/DotNetWorker/Converters/EnumerableConverter.cs index 344be1fbe..b6603a704 100644 --- a/src/DotNetWorker/Converters/EnumerableConverter.cs +++ b/src/DotNetWorker/Converters/EnumerableConverter.cs @@ -18,6 +18,7 @@ internal class EnumerableConverter : IConverter public bool TryConvert(ConverterContext context, out object? target) { EnumerableTargetType? targetType = null; + target = null; // Array if (context.Parameter.Type.IsArray) { @@ -38,49 +39,40 @@ public bool TryConvert(ConverterContext context, out object? target) // Only apply if user is requesting an array, list, or hashset if (targetType is not null) - { + { // Valid options from FunctionRpc.proto are string, byte, double and long collection - if (context.Source is IEnumerable enumerableString) - { - target = GetTarget(enumerableString, targetType); - return true; - } - else if (context.Source is IEnumerable enumerableBytes) - { - target = GetTarget(enumerableBytes, targetType); - return true; - } - else if (context.Source is IEnumerable enumerableDouble) - { - target = GetTarget(enumerableDouble, targetType); - return true; - } - else if (context.Source is IEnumerable enumerableLong) + target = context.Source switch { - target = GetTarget(enumerableLong, targetType); - return true; - } + IEnumerable source => GetTarget(source, targetType), + IEnumerable source => GetTarget(source, targetType), + IEnumerable source => GetTarget(source, targetType), + IEnumerable source => GetTarget(source, targetType), + _ => null + }; } - target = default; - return false; + if (target is null) + { + target = default; + return false; + } + else + { + return true; + } } // Dictionary and Lookup not handled because we don't know // what they keySelector and elementSelector should be. private static object? GetTarget(IEnumerable source, EnumerableTargetType? targetType) { - switch (targetType) + return targetType switch { - case EnumerableTargetType.Array: - return source.ToArray(); - case EnumerableTargetType.HashSet: - return source.ToHashSet(); - case EnumerableTargetType.List: - return source.ToList(); - default: - return null; - } + EnumerableTargetType.Array => source.ToArray(), + EnumerableTargetType.HashSet => source.ToHashSet(), + EnumerableTargetType.List => source.ToList(), + _ => null, + }; } private enum EnumerableTargetType From c5f11856b11ff233dbdfdb1333509b52218e850a Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 10:33:01 -0800 Subject: [PATCH 11/24] minor updates from cr --- sdk/Sdk/Constants.cs | 1 + sdk/Sdk/FunctionMetadataGenerator.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs index a00156f83..8d6cf92ff 100644 --- a/sdk/Sdk/Constants.cs +++ b/sdk/Sdk/Constants.cs @@ -32,3 +32,4 @@ internal static class Constants internal const string HttpTriggerBindingType = "HttpTrigger"; internal const string IsBatchedKey = "isbatched"; } +} diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 2914e06c9..d99614d53 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -327,7 +327,7 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe private static TypeReference? GetTaskElementType(TypeReference typeReference) { - if (typeReference is null || typeReference.FullName == Constants.TaskType) + if (typeReference is null || string.Equals(typeReference.FullName, Constants.TaskType, StringComparison.Ordinal)) { return null; } @@ -418,7 +418,7 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a else { throw new FunctionsMetadataGenerationException("Function is configured to process events in batches but parameter type is not iterable. " + - $"Change parameter { "'" + parameterName + "'" ?? "type" } to be an IEnumerable type or set 'IsBatched' to false on your '{attribute.AttributeType.Name.Replace("Attribute", "")}' attribute."); + $"Change parameter named '{ parameterName }' to be an IEnumerable type or set 'IsBatched' to false on your '{attribute.AttributeType.Name.Replace("Attribute", "")}' attribute."); } } // Batching set to false From b54d2549576b2049345162660811220b93cd9513 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 13:19:07 -0800 Subject: [PATCH 12/24] change where we get default value --- .../DefaultValueAttribute.cs | 29 +++++++++++++++++++ .../IBatchedInput.cs | 5 ++-- .../EventHubTriggerAttribute.cs | 4 +-- sdk/Sdk/Constants.cs | 1 + sdk/Sdk/CustomAttributeExtensions.cs | 28 ++++++++++++++++-- sdk/Sdk/FunctionMetadataGenerator.cs | 1 - 6 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs diff --git a/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs new file mode 100644 index 000000000..606a073f9 --- /dev/null +++ b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker +{ + [AttributeUsage(AttributeTargets.Property)] + public class DefaultValueAttribute : Attribute + { + /// + /// Define a default value for a property on a FunctionAttribute type. + /// + /// + public DefaultValueAttribute(string stringDefault) + { + DefaultStringValue = stringDefault; + } + + public DefaultValueAttribute(bool boolDefault) + { + DefaultBoolValue = boolDefault; + } + + public string? DefaultStringValue { get; } + + public bool? DefaultBoolValue { get; } + } +} diff --git a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs index cbff0f94e..a3263beee 100644 --- a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs +++ b/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Generic; namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions { @@ -13,9 +14,7 @@ public interface IBatchedInput /// true => "Many" /// false => "One" /// - /// To default to a particular true or false, the constructor of the attribute that inherits - /// from this must include 'bool isBatched = [true / false]' as an optional parameters - /// on the constructor + /// To default to a particular true or false, add the "DefaultValue" attribute to the method. /// public bool IsBatched { get; set; } } diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index a51e05c97..15113e5ee 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -11,10 +11,9 @@ public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, IBatched /// Create an instance of this attribute. /// /// Event hub to listen on for messages. - public EventHubTriggerAttribute(string eventHubName, bool IsBatched = true) + public EventHubTriggerAttribute(string eventHubName) { EventHubName = eventHubName; - this.IsBatched = IsBatched; } /// @@ -35,6 +34,7 @@ public EventHubTriggerAttribute(string eventHubName, bool IsBatched = true) /// /// Gets or sets the configuration to enable batch processing of events. Default value is "true". /// + [DefaultValue(true)] public bool IsBatched { get; set; } } } diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs index 8d6cf92ff..01618dcf1 100644 --- a/sdk/Sdk/Constants.cs +++ b/sdk/Sdk/Constants.cs @@ -11,6 +11,7 @@ internal static class Constants internal const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; internal const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; internal const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; + internal const string DefaultValueAttributeType = "Microsoft.Azure.Functions.Worker.DefaultValueAttribute"; // System types internal const string IEnumerableType = "System.Collections.IEnumerable"; diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index 535911f8b..bb7d43f2c 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -14,16 +14,40 @@ public static class CustomAttributeExtensions public static IDictionary GetAllDefinedProperties(this CustomAttribute attribute) { var properties = new Dictionary(); - // To avoid needing to instantiate any types, assume that the constructor // argument names are equal to property names. + LoadDefaultProperties(properties, attribute); LoadConstructorArguments(properties, attribute); LoadDefinedProperties(properties, attribute); return properties; } - private static void LoadConstructorArguments(IDictionary properties, CustomAttribute attribute) + private static IEnumerable<(string, CustomAttributeArgument?)> GetDefaultValues(this CustomAttribute attribute) + { + return attribute.AttributeType.Resolve().Properties + .Select(p => (p.Name, p.CustomAttributes + .Where(attribute => string.Equals(attribute.AttributeType.FullName, Constants.DefaultValueAttributeType, StringComparison.Ordinal)) + .SingleOrDefault() + ?.ConstructorArguments.SingleOrDefault())) + .Where(t => t.Item2 is not null); + } + + private static void LoadDefaultProperties(IDictionary properties, CustomAttribute attribute) + { + var propertyDefaults = attribute.GetDefaultValues(); + + for (int i = 0; i < propertyDefaults.Count(); i++) + { + var propertyDefault = propertyDefaults.ElementAt(i); + if (propertyDefault.Item2 is not null) + { + properties[propertyDefault.Item1] = propertyDefault.Item2.Value.Value; + } + } + } + + private static void LoadConstructorArguments(IDictionary properties, CustomAttribute attribute) { var constructorParams = attribute.Constructor.Resolve().Parameters; for (int i = 0; i < attribute.ConstructorArguments.Count; i++) diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index d99614d53..79ca560e3 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -395,7 +395,6 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a // "isbatched": true => "Cardinality": "Many" // "isbatched": false => "Cardinality": "One" // TODO: do not rely on property alone - if (bindingDict.TryGetValue(Constants.IsBatchedKey, out object isBatchedValue) && isBatchedValue is bool isBatched) { From 8ce8fce4d7ffcdf0c2488c3fc520c08a49347add Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 13:22:38 -0800 Subject: [PATCH 13/24] casing --- sdk/Sdk/CustomAttributeExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index bb7d43f2c..e70e2bf4d 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -42,7 +42,7 @@ private static void LoadDefaultProperties(IDictionary properties var propertyDefault = propertyDefaults.ElementAt(i); if (propertyDefault.Item2 is not null) { - properties[propertyDefault.Item1] = propertyDefault.Item2.Value.Value; + properties[propertyDefault.Item1.ToLower()] = propertyDefault.Item2.Value.Value; } } } From 182e9a4214f3c3041f377715f55981e92ef0ecae Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 13:43:24 -0800 Subject: [PATCH 14/24] fix docs --- extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs | 2 +- .../Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs index ab36b0dd1..c5d99700b 100644 --- a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs @@ -96,7 +96,7 @@ public KafkaTriggerAttribute(string brokerList, string topic) public string? SslKeyPassword { get; set; } /// - /// Gets or sets the configuration to enable batch processing of events. Default value is "true". + /// Gets or sets the configuration to enable batch processing of events. Default value is "false". /// public bool IsBatched { get; set; } } diff --git a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs index 1c54fb422..8bfa6c993 100644 --- a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs +++ b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs @@ -69,7 +69,7 @@ public string? SubscriptionName public bool IsSessionsEnabled { get; set; } /// - /// Gets or sets the configuration to enable batch processing of events. Default value is "true". + /// Gets or sets the configuration to enable batch processing of events. Default value is "false". /// public bool IsBatched { get; set; } } From 887629780c13b24f268f0497d0df15a9815fe885 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 16:17:55 -0800 Subject: [PATCH 15/24] Modify batched output to be ISupportCardinality - set up for extensible future --- ...BatchedInput.cs => ISupportCardinality.cs} | 10 ++++-- .../EventHubTriggerAttribute.cs | 36 +++++++++++++++++-- .../KafkaTriggerAttribute.cs | 36 +++++++++++++++++-- .../ServiceBusTriggerAttribute.cs | 36 +++++++++++++++++-- 4 files changed, 110 insertions(+), 8 deletions(-) rename extensions/Worker.Extensions.Abstractions/{IBatchedInput.cs => ISupportCardinality.cs} (80%) diff --git a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs b/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs similarity index 80% rename from extensions/Worker.Extensions.Abstractions/IBatchedInput.cs rename to extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs index a3263beee..653554e51 100644 --- a/extensions/Worker.Extensions.Abstractions/IBatchedInput.cs +++ b/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs @@ -6,7 +6,7 @@ namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions { - public interface IBatchedInput + public interface ISupportCardinality { /// /// Configures trigger to process events in batches or one at a time. @@ -16,6 +16,12 @@ public interface IBatchedInput /// /// To default to a particular true or false, add the "DefaultValue" attribute to the method. /// - public bool IsBatched { get; set; } + public Cardinality Cardinality { get; set; } + } + + public enum Cardinality + { + Many, + One } } diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index 15113e5ee..23510aacc 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -5,8 +5,10 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, IBatchedInput + public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + private bool _isBatched = true; + /// /// Create an instance of this attribute. /// @@ -35,6 +37,36 @@ public EventHubTriggerAttribute(string eventHubName) /// Gets or sets the configuration to enable batch processing of events. Default value is "true". /// [DefaultValue(true)] - public bool IsBatched { get; set; } + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + + Cardinality ISupportCardinality.Cardinality + { + get + { + if (_isBatched) + { + return Cardinality.Many; + } + else + { + return Cardinality.One; + } + } + set + { + if (value.Equals(Cardinality.Many)) + { + _isBatched = true; + } + else + { + _isBatched = false; + } + } + } } } diff --git a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs index c5d99700b..bd2d214cd 100644 --- a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs @@ -8,8 +8,10 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class KafkaTriggerAttribute : TriggerBindingAttribute, IBatchedInput + public sealed class KafkaTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + private bool _isBatched = false; + public KafkaTriggerAttribute(string brokerList, string topic) { BrokerList = brokerList; @@ -98,6 +100,36 @@ public KafkaTriggerAttribute(string brokerList, string topic) /// /// Gets or sets the configuration to enable batch processing of events. Default value is "false". /// - public bool IsBatched { get; set; } + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + + Cardinality ISupportCardinality.Cardinality + { + get + { + if (_isBatched) + { + return Cardinality.Many; + } + else + { + return Cardinality.One; + } + } + set + { + if (value.Equals(Cardinality.Many)) + { + _isBatched = true; + } + else + { + _isBatched = false; + } + } + } } } diff --git a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs index 8bfa6c993..35ef6f461 100644 --- a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs +++ b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs @@ -5,8 +5,10 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class ServiceBusTriggerAttribute : TriggerBindingAttribute, IBatchedInput + public sealed class ServiceBusTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + private bool _isBatched = false; + private readonly string? _queueName; private readonly string? _topicName; private readonly string? _subscriptionName; @@ -71,6 +73,36 @@ public string? SubscriptionName /// /// Gets or sets the configuration to enable batch processing of events. Default value is "false". /// - public bool IsBatched { get; set; } + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + + Cardinality ISupportCardinality.Cardinality + { + get + { + if (_isBatched) + { + return Cardinality.Many; + } + else + { + return Cardinality.One; + } + } + set + { + if (value.Equals(Cardinality.Many)) + { + _isBatched = true; + } + else + { + _isBatched = false; + } + } + } } } From 8d0d9a17b26a1df0f81af414d1c38e254b33a1b0 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 18:22:10 -0800 Subject: [PATCH 16/24] array converter only --- .../ISupportCardinality.cs | 11 ++- .../EventHubTriggerAttribute.cs | 1 + sdk/Sdk/Constants.cs | 1 + sdk/Sdk/FunctionMetadataGenerator.cs | 20 +++-- .../Features/GrpcFunctionBindingsFeature.cs | 2 +- src/DotNetWorker/Converters/ArrayConverter.cs | 63 ++++++++++++++ .../Converters/EnumerableConverter.cs | 87 ------------------- .../Hosting/ServiceCollectionExtensions.cs | 2 +- ...nverterTests.cs => ArrayConverterTests.cs} | 29 ++----- 9 files changed, 93 insertions(+), 123 deletions(-) create mode 100644 src/DotNetWorker/Converters/ArrayConverter.cs delete mode 100644 src/DotNetWorker/Converters/EnumerableConverter.cs rename test/DotNetWorkerTests/Converters/{EnumerableConverterTests.cs => ArrayConverterTests.cs} (67%) diff --git a/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs b/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs index 653554e51..15b4fce77 100644 --- a/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs +++ b/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs @@ -9,12 +9,11 @@ namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions public interface ISupportCardinality { /// - /// Configures trigger to process events in batches or one at a time. - /// This translates to values for the "cardinality" property in WebJobs terms. - /// true => "Many" - /// false => "One" - /// - /// To default to a particular true or false, add the "DefaultValue" attribute to the method. + /// Configures the "cardinality" property + /// This property indicates that the requested type is an array. Note that for + /// inputs and outputs, the effect of cardinality may be different (ex: electing to + /// receive a collection of events vs. indicating that my return type will be + /// a collection). /// public Cardinality Cardinality { get; set; } } diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index 23510aacc..1b350eba9 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Functions.Worker { public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + // Batch by default private bool _isBatched = true; /// diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs index 01618dcf1..faa78e00b 100644 --- a/sdk/Sdk/Constants.cs +++ b/sdk/Sdk/Constants.cs @@ -28,6 +28,7 @@ internal static class Constants internal const string TaskGenericType = "System.Threading.Tasks.Task`1"; internal const string TaskType = "System.Threading.Tasks.Task"; internal const string VoidType = "System.Void"; + internal const string ReadOnlyMemoryOfBytes = "System.ReadOnlyMemory`1"; internal const string ReturnBindingName = "$return"; internal const string HttpTriggerBindingType = "HttpTrigger"; diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 79ca560e3..4dee3ff6a 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -389,12 +389,18 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a } // Determine if we should set the "Cardinality" property based on - // the presence of "isBatched." This is a property that is from the IBatchedInput - // interface. + // the presence of "isBatched." This is a property that is from the + // attributes that implement the ISupportCardinality interface. + // + // Note that we are directly looking for "IsBatched" today while we + // are not actually instantiating the Attribute type and instead relying + // on type inspection via Mono.Cecil. + // TODO: Do not hard-code "isBatched" as the property to set cardinality. + // We should rely on the interface + // // Conversion rule // "isbatched": true => "Cardinality": "Many" // "isbatched": false => "Cardinality": "One" - // TODO: do not rely on property alone if (bindingDict.TryGetValue(Constants.IsBatchedKey, out object isBatchedValue) && isBatchedValue is bool isBatched) { @@ -402,7 +408,10 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a if (isBatched) { bindingDict["Cardinality"] = "Many"; - // Throw if parameter type is not IEnumerable + // Throw if parameter type is *definitely* not a collection type. + // Note that this logic doesn't dictate what we can/can't do, and + // we can be more restrictive in the future because today some + // scenarios result in runtime failures. if (IsIterableCollection(parameterType, out DataType dataType)) { if (dataType.Equals(DataType.String)) @@ -586,7 +595,8 @@ private static bool IsStringType(string fullName) private static bool IsBinaryType(string fullName) { - return string.Equals(fullName, Constants.ByteArrayType, StringComparison.Ordinal); + return string.Equals(fullName, Constants.ByteArrayType, StringComparison.Ordinal) + || string.Equals(fullName, Constants.ReadOnlyMemoryOfBytes, StringComparison.Ordinal); } private static string GetBindingType(CustomAttribute attribute) diff --git a/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs b/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs index e30d9868b..b687bfbf3 100644 --- a/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs +++ b/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs @@ -117,7 +117,7 @@ public void SetOutputBinding(string name, object value) TypedData.DataOneofCase.Json => typedData.Json, TypedData.DataOneofCase.Bytes => typedData.Bytes.Memory, TypedData.DataOneofCase.CollectionBytes => typedData.CollectionBytes.Bytes.Select(element => { - return element.Memory.ToArray(); + return element.Memory; }), TypedData.DataOneofCase.CollectionString => typedData.CollectionString.String, TypedData.DataOneofCase.CollectionDouble => typedData.CollectionDouble.Double, diff --git a/src/DotNetWorker/Converters/ArrayConverter.cs b/src/DotNetWorker/Converters/ArrayConverter.cs new file mode 100644 index 000000000..46a1777f1 --- /dev/null +++ b/src/DotNetWorker/Converters/ArrayConverter.cs @@ -0,0 +1,63 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Azure.Functions.Worker.Converters +{ + // Converting IEnumerable<> to Array + internal class ArrayConverter : IConverter + { + // Convert IEnumerable to array + public bool TryConvert(ConverterContext context, out object? target) + { + // Ensure requested type is an array + if (context.Parameter.Type.IsArray) + { + Type? elementType = context.Parameter.Type.GetElementType(); + if (elementType is not null) + { + // Ensure that we can assign from source to parameter type + if (elementType.IsAssignableFrom(typeof(string)) + || elementType.IsAssignableFrom(typeof(byte[])) + || elementType.IsAssignableFrom(typeof(ReadOnlyMemory)) + || elementType.IsAssignableFrom(typeof(long)) + || elementType.IsAssignableFrom(typeof(double))) + { + target = null; + target = context.Source switch + { + IEnumerable source => source.ToArray(), + IEnumerable> source => GetBinaryData(source, elementType!), + IEnumerable source => source.ToArray(), + IEnumerable source => source.ToArray(), + _ => null + }; + + if (target is not null) + { + return true; + } + } + } + } + + target = default; + return false; + } + + private static object? GetBinaryData(IEnumerable> source, Type targetType) + { + if (targetType.IsAssignableFrom(typeof(ReadOnlyMemory))) + { + return source.ToArray(); + } + else + { + return source.Select(i => i.ToArray()); + } + } + } +} diff --git a/src/DotNetWorker/Converters/EnumerableConverter.cs b/src/DotNetWorker/Converters/EnumerableConverter.cs deleted file mode 100644 index b6603a704..000000000 --- a/src/DotNetWorker/Converters/EnumerableConverter.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.Azure.Functions.Worker.Converters -{ - // Converting IEnumerable<> to Array - internal class EnumerableConverter : IConverter - { - private static Type ListType = typeof(List<>); - private static Type HashSetType = typeof(HashSet<>); - - // Convert IEnumerable from common types. IEnumerable types will - // be converted by TypeConverter. - public bool TryConvert(ConverterContext context, out object? target) - { - EnumerableTargetType? targetType = null; - target = null; - // Array - if (context.Parameter.Type.IsArray) - { - targetType = EnumerableTargetType.Array; - } - // List or HashSet - else if (context.Parameter.Type.IsGenericType) - { - if (context.Parameter.Type.GetGenericTypeDefinition().IsAssignableFrom(ListType)) - { - targetType = EnumerableTargetType.List; - } - else if (context.Parameter.Type.GetGenericTypeDefinition().IsAssignableFrom(HashSetType)) - { - targetType = EnumerableTargetType.HashSet; - } - } - - // Only apply if user is requesting an array, list, or hashset - if (targetType is not null) - { - // Valid options from FunctionRpc.proto are string, byte, double and long collection - target = context.Source switch - { - IEnumerable source => GetTarget(source, targetType), - IEnumerable source => GetTarget(source, targetType), - IEnumerable source => GetTarget(source, targetType), - IEnumerable source => GetTarget(source, targetType), - _ => null - }; - } - - if (target is null) - { - target = default; - return false; - } - else - { - return true; - } - } - - // Dictionary and Lookup not handled because we don't know - // what they keySelector and elementSelector should be. - private static object? GetTarget(IEnumerable source, EnumerableTargetType? targetType) - { - return targetType switch - { - EnumerableTargetType.Array => source.ToArray(), - EnumerableTargetType.HashSet => source.ToHashSet(), - EnumerableTargetType.List => source.ToList(), - _ => null, - }; - } - - private enum EnumerableTargetType - { - Array, - List, - Dictionary, - HashSet, - Lookup - } - } -} diff --git a/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs b/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs index 6b8fef7cc..9f8698495 100644 --- a/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs +++ b/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs @@ -110,7 +110,7 @@ internal static IServiceCollection RegisterDefaultConverters(this IServiceCollec .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton(); } internal static IServiceCollection RegisterOutputChannel(this IServiceCollection services) diff --git a/test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs b/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs similarity index 67% rename from test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs rename to test/DotNetWorkerTests/Converters/ArrayConverterTests.cs index 0c5b27716..fd8376b49 100644 --- a/test/DotNetWorkerTests/Converters/EnumerableConverterTests.cs +++ b/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs @@ -10,14 +10,14 @@ namespace Microsoft.Azure.Functions.Worker.Tests.Converters { - public class EnumerableConverterTests + public class ArrayConverterTests { private const string _sourceString = "hello"; private static readonly byte[] _sourceBytes = Encoding.UTF8.GetBytes(_sourceString); private static readonly ReadOnlyMemory _sourceMemory = new ReadOnlyMemory(_sourceBytes); - private EnumerableConverter _converter = new EnumerableConverter(); + private ArrayConverter _converter = new ArrayConverter(); - private static readonly IEnumerable _sourceMemoryEnumerable = new RepeatedField() { _sourceBytes }; + private static readonly IEnumerable> _sourceMemoryEnumerable = new RepeatedField>() { _sourceMemory }; private static readonly RepeatedField _sourceStringEnumerable = new RepeatedField() { _sourceString }; private static readonly RepeatedField _sourceDoubleEnumerable = new RepeatedField() { 1.0 }; private static readonly RepeatedField _sourceLongEnumerable = new RepeatedField() { 2000 }; @@ -31,28 +31,11 @@ public void ConvertCollectionBytesToByteDoubleArray() } [Fact] - public void ConvertCollectionBytesToByteArrayList() + public void ConvertCollectionBytesToReadOnlyByteArray() { - var context = new TestConverterContext("output", typeof(List), _sourceMemoryEnumerable); + var context = new TestConverterContext("output", typeof(ReadOnlyMemory[]), _sourceMemoryEnumerable); Assert.True(_converter.TryConvert(context, out object target)); - TestUtility.AssertIsTypeAndConvert>(target); - } - - [Fact] - public void ConvertCollectionBytesToByteArrayHashSet() - { - var context = new TestConverterContext("output", typeof(HashSet), _sourceMemoryEnumerable); - Assert.True(_converter.TryConvert(context, out object target)); - TestUtility.AssertIsTypeAndConvert>(target); - } - - // Queue implements IEnumerable but is not converted - [Fact] - public void ConvertCollectionBytesToByteArrayQueue() - { - var context = new TestConverterContext("output", typeof(Queue), _sourceMemoryEnumerable); - Assert.False(context.Parameter.Type.IsAssignableFrom(typeof(IEnumerable))); - Assert.False(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert[]>(target); } [Fact] From 4f597208b56fd2f5d458dd1cc7d6869fe73d9c63 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 18:53:47 -0800 Subject: [PATCH 17/24] test fixes --- sdk/Sdk/Constants.cs | 2 +- sdk/Sdk/CustomAttributeExtensions.cs | 6 +++--- sdk/Sdk/FunctionMetadataGenerator.cs | 8 ++++---- src/DotNetWorker/Converters/ArrayConverter.cs | 2 +- .../FunctionMetadataGeneratorTests.cs | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs index faa78e00b..c848750eb 100644 --- a/sdk/Sdk/Constants.cs +++ b/sdk/Sdk/Constants.cs @@ -32,6 +32,6 @@ internal static class Constants internal const string ReturnBindingName = "$return"; internal const string HttpTriggerBindingType = "HttpTrigger"; - internal const string IsBatchedKey = "isbatched"; + internal const string IsBatchedKey = "IsBatched"; } } diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index e70e2bf4d..6951102bc 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -42,7 +42,7 @@ private static void LoadDefaultProperties(IDictionary properties var propertyDefault = propertyDefaults.ElementAt(i); if (propertyDefault.Item2 is not null) { - properties[propertyDefault.Item1.ToLower()] = propertyDefault.Item2.Value.Value; + properties[propertyDefault.Item1] = propertyDefault.Item2.Value.Value; } } } @@ -65,7 +65,7 @@ private static void LoadConstructorArguments(IDictionary propert paramValue = GetEnrichedValue(param!.ParameterType, paramValue); - properties[paramName.ToLower()] = paramValue!; + properties[paramName] = paramValue!; } } @@ -83,7 +83,7 @@ private static void LoadDefinedProperties(IDictionary properties propVal = GetEnrichedValue(property.Argument.Type, propVal); - properties[propName.ToLower()] = propVal!; + properties[propName] = propVal!; } } diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index 4dee3ff6a..fed0dbea4 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -389,18 +389,18 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a } // Determine if we should set the "Cardinality" property based on - // the presence of "isBatched." This is a property that is from the + // the presence of "IsBatched." This is a property that is from the // attributes that implement the ISupportCardinality interface. // // Note that we are directly looking for "IsBatched" today while we // are not actually instantiating the Attribute type and instead relying // on type inspection via Mono.Cecil. - // TODO: Do not hard-code "isBatched" as the property to set cardinality. + // TODO: Do not hard-code "IsBatched" as the property to set cardinality. // We should rely on the interface // // Conversion rule - // "isbatched": true => "Cardinality": "Many" - // "isbatched": false => "Cardinality": "One" + // "IsBatched": true => "Cardinality": "Many" + // "IsBatched": false => "Cardinality": "One" if (bindingDict.TryGetValue(Constants.IsBatchedKey, out object isBatchedValue) && isBatchedValue is bool isBatched) { diff --git a/src/DotNetWorker/Converters/ArrayConverter.cs b/src/DotNetWorker/Converters/ArrayConverter.cs index 46a1777f1..ab3db09fa 100644 --- a/src/DotNetWorker/Converters/ArrayConverter.cs +++ b/src/DotNetWorker/Converters/ArrayConverter.cs @@ -56,7 +56,7 @@ public bool TryConvert(ConverterContext context, out object? target) } else { - return source.Select(i => i.ToArray()); + return source.Select(i => i.ToArray()).ToArray(); } } } diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index a4494479b..1d2aa5aac 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -412,7 +412,7 @@ public void MultiOutput_OnMethod_Throws() var module = ModuleDefinition.ReadModule(_thisAssembly.Location); var typeDef = TestUtility.GetTypeDefinition(typeof(MultiOutput_Method)); - var exception = Assert.Throws(() => generator.GenerateFunctionMetadata(typeDef)); + var exception = Assert.Throws(() => generator.GenerateFunctionMetadata(typeDef)); Assert.Contains($"Found multiple Output bindings on method", exception.Message); } @@ -468,8 +468,8 @@ void ValidateTrigger(ExpandoObject b, bool many) { "Name", "input" }, { "Type", "EventHubTrigger" }, { "Direction", "In" }, - { "eventhubname", "test" }, - { "connection", "EventHubConnectionAppSetting" } + { "eventHubName", "test" }, + { "Connection", "EventHubConnectionAppSetting" } }; if (many) From d0db91d6637363ae008d4efa6091365c0628efa6 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 20:25:34 -0800 Subject: [PATCH 18/24] release notes --- release_notes.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/release_notes.md b/release_notes.md index 9587e77cb..ca63452ea 100644 --- a/release_notes.md +++ b/release_notes.md @@ -6,3 +6,13 @@ - Updates to HttpResponseData - API updates - Support for response Cookies +- Add support for batched trigger events (#205) + - The following services allow trigger events to be batched: + - Event Hubs (batched by default) + - Service Bus (set `IsBatched = true` in trigger attribute) + - Kafka (set `IsBatched = true` in trigger attribute) + - To read batched event data in function code: + - Use array (`[]`), `IList`, `ICollection`, or `IEnumerable` if event data resolves to a primitive type (`string`, `int`, `byte[]`, `long`, `double`). + - `byte[]` can also be read as an `ReadOnlyMemory`. This is the more performant option, especially for large payloads. + - Use a class that implements `IEnumerable` or `IEnumerable` for POCO event data (example: `List`). +- Fail function execution if the requested parameter cannot be converted to the specified type (#216) \ No newline at end of file From b734b7fb5557a4abc353b5c70cc813380be436f6 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 22:12:02 -0800 Subject: [PATCH 19/24] update array converter logic --- src/DotNetWorker/Converters/ArrayConverter.cs | 20 +++++++------------ .../Converters/ArrayConverterTests.cs | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/DotNetWorker/Converters/ArrayConverter.cs b/src/DotNetWorker/Converters/ArrayConverter.cs index ab3db09fa..a876563e5 100644 --- a/src/DotNetWorker/Converters/ArrayConverter.cs +++ b/src/DotNetWorker/Converters/ArrayConverter.cs @@ -13,6 +13,7 @@ internal class ArrayConverter : IConverter // Convert IEnumerable to array public bool TryConvert(ConverterContext context, out object? target) { + target = null; // Ensure requested type is an array if (context.Parameter.Type.IsArray) { @@ -20,13 +21,12 @@ public bool TryConvert(ConverterContext context, out object? target) if (elementType is not null) { // Ensure that we can assign from source to parameter type - if (elementType.IsAssignableFrom(typeof(string)) - || elementType.IsAssignableFrom(typeof(byte[])) - || elementType.IsAssignableFrom(typeof(ReadOnlyMemory)) - || elementType.IsAssignableFrom(typeof(long)) - || elementType.IsAssignableFrom(typeof(double))) + if (elementType.Equals(typeof(string)) + || elementType.Equals(typeof(byte[])) + || elementType.Equals(typeof(ReadOnlyMemory)) + || elementType.Equals(typeof(long)) + || elementType.Equals(typeof(double))) { - target = null; target = context.Source switch { IEnumerable source => source.ToArray(), @@ -35,17 +35,11 @@ public bool TryConvert(ConverterContext context, out object? target) IEnumerable source => source.ToArray(), _ => null }; - - if (target is not null) - { - return true; - } } } } - target = default; - return false; + return target is not null; } private static object? GetBinaryData(IEnumerable> source, Type targetType) diff --git a/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs b/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs index fd8376b49..d6b2be5be 100644 --- a/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs +++ b/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs @@ -23,7 +23,7 @@ public class ArrayConverterTests private static readonly RepeatedField _sourceLongEnumerable = new RepeatedField() { 2000 }; [Fact] - public void ConvertCollectionBytesToByteDoubleArray() + public void ConvertCollectionBytesToJaggedByteArray() { var context = new TestConverterContext("output", typeof(byte[][]), _sourceMemoryEnumerable); Assert.True(_converter.TryConvert(context, out object target)); From 775d70ede1223fdf8ed4823a2e27947faa278b98 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 23:02:35 -0800 Subject: [PATCH 20/24] remove nested type search and update readme because double and int aren't passed over by host yet apparently --- release_notes.md | 6 +++--- sdk/Sdk/CustomAttributeExtensions.cs | 3 +-- sdk/Sdk/FunctionMetadataGenerator.cs | 7 +------ 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/release_notes.md b/release_notes.md index ca63452ea..173837c12 100644 --- a/release_notes.md +++ b/release_notes.md @@ -12,7 +12,7 @@ - Service Bus (set `IsBatched = true` in trigger attribute) - Kafka (set `IsBatched = true` in trigger attribute) - To read batched event data in function code: - - Use array (`[]`), `IList`, `ICollection`, or `IEnumerable` if event data resolves to a primitive type (`string`, `int`, `byte[]`, `long`, `double`). - - `byte[]` can also be read as an `ReadOnlyMemory`. This is the more performant option, especially for large payloads. - - Use a class that implements `IEnumerable` or `IEnumerable` for POCO event data (example: `List`). + - Use array (`[]`), `IList`, `ICollection`, or `IEnumerable` if event data is `string`, `byte[]`, or `ReadOnlyMemory`. + - Note: `ReadOnlyMemory` is the more performant option to read binary data, especially for large payloads. + - Use a class that implements `IEnumerable` or `IEnumerable` for serializable event data (example: `List`). - Fail function execution if the requested parameter cannot be converted to the specified type (#216) \ No newline at end of file diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index 6951102bc..0f0b2e745 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -37,9 +37,8 @@ private static void LoadDefaultProperties(IDictionary properties { var propertyDefaults = attribute.GetDefaultValues(); - for (int i = 0; i < propertyDefaults.Count(); i++) + foreach (var propertyDefault in propertyDefaults) { - var propertyDefault = propertyDefaults.ElementAt(i); if (propertyDefault.Item2 is not null) { properties[propertyDefault.Item1] = propertyDefault.Item2.Value.Value; diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index fed0dbea4..d74f80117 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -510,8 +510,7 @@ private static bool IsDerivedFrom(TypeDefinition definition, string interfaceFul private static bool HasInterface(TypeDefinition definition, string interfaceFullName) { - return definition.Interfaces.Any(i => string.Equals(i.InterfaceType.FullName, interfaceFullName, StringComparison.Ordinal)) - || definition.NestedTypes.Any(t => IsDerivedFrom(t, interfaceFullName)); + return definition.Interfaces.Any(i => string.Equals(i.InterfaceType.FullName, interfaceFullName, StringComparison.Ordinal)); } private static bool IsSubclassOf(TypeDefinition definition, string interfaceFullName) @@ -567,10 +566,6 @@ private static bool IsSubclassOf(TypeDefinition definition, string interfaceFull .Select(i => ResolveIEnumerableOfTType(i.InterfaceType, foundMapping)) .Where(name => name is not null) .FirstOrDefault() - ?? definition.NestedTypes - .Select(t => ResolveIEnumerableOfTType(t, foundMapping)) - .Where(name => name is not null) - .FirstOrDefault() ?? ResolveIEnumerableOfTType(definition.BaseType, foundMapping); } From 8fb62e2e58752cad1f890fb369974ac5cf1384ad Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 23:06:11 -0800 Subject: [PATCH 21/24] address comment to make default value attribute (a temporary attribute) less discoverable --- .../Worker.Extensions.Abstractions/DefaultValueAttribute.cs | 2 +- .../Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs | 1 + sdk/Sdk/Constants.cs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs index 606a073f9..8f9a0508e 100644 --- a/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs +++ b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Azure.Functions.Worker +namespace Microsoft.Azure.Functions.Worker.Annotations { [AttributeUsage(AttributeTargets.Property)] public class DefaultValueAttribute : Attribute diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index 1b350eba9..7d570a472 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Annotations; namespace Microsoft.Azure.Functions.Worker { diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs index c848750eb..3c5c11131 100644 --- a/sdk/Sdk/Constants.cs +++ b/sdk/Sdk/Constants.cs @@ -11,7 +11,7 @@ internal static class Constants internal const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; internal const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; internal const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; - internal const string DefaultValueAttributeType = "Microsoft.Azure.Functions.Worker.DefaultValueAttribute"; + internal const string DefaultValueAttributeType = "Microsoft.Azure.Functions.Worker.Annotations.DefaultValueAttribute"; // System types internal const string IEnumerableType = "System.Collections.IEnumerable"; From 37de7a8ca5c50648bbd1513bfdc622af23317a78 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 23:12:43 -0800 Subject: [PATCH 22/24] nit: add example to readme --- release_notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_notes.md b/release_notes.md index 173837c12..85979f321 100644 --- a/release_notes.md +++ b/release_notes.md @@ -12,7 +12,7 @@ - Service Bus (set `IsBatched = true` in trigger attribute) - Kafka (set `IsBatched = true` in trigger attribute) - To read batched event data in function code: - - Use array (`[]`), `IList`, `ICollection`, or `IEnumerable` if event data is `string`, `byte[]`, or `ReadOnlyMemory`. + - Use array (`[]`), `IList`, `ICollection`, or `IEnumerable` if event data is `string`, `byte[]`, or `ReadOnlyMemory` (example: `string[]`). - Note: `ReadOnlyMemory` is the more performant option to read binary data, especially for large payloads. - Use a class that implements `IEnumerable` or `IEnumerable` for serializable event data (example: `List`). - Fail function execution if the requested parameter cannot be converted to the specified type (#216) \ No newline at end of file From ea58fa3da4b2d0117a03b52b1bdae474bdf39a58 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 23:28:31 -0800 Subject: [PATCH 23/24] update namespace --- .../Worker.Extensions.Abstractions/DefaultValueAttribute.cs | 2 +- sdk/Sdk/CustomAttributeExtensions.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs index 8f9a0508e..189fb7027 100644 --- a/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs +++ b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.Azure.Functions.Worker.Annotations +namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions { [AttributeUsage(AttributeTargets.Property)] public class DefaultValueAttribute : Attribute diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index 0f0b2e745..3f9f3d205 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -62,8 +62,7 @@ private static void LoadConstructorArguments(IDictionary propert continue; } - paramValue = GetEnrichedValue(param!.ParameterType, paramValue); - + paramValue = GetEnrichedValue(param!.ParameterType, paramValue); properties[paramName] = paramValue!; } } From 13594cd35ce5a3a1f25b6e035b379eb640e8bb10 Mon Sep 17 00:00:00 2001 From: Marie Hoeger Date: Thu, 4 Mar 2021 23:42:19 -0800 Subject: [PATCH 24/24] namespace reference --- .../Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs | 1 - sdk/Sdk/Constants.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index 7d570a472..1b350eba9 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; -using Microsoft.Azure.Functions.Worker.Annotations; namespace Microsoft.Azure.Functions.Worker { diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs index 3c5c11131..1f3b6dd15 100644 --- a/sdk/Sdk/Constants.cs +++ b/sdk/Sdk/Constants.cs @@ -11,7 +11,7 @@ internal static class Constants internal const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; internal const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; internal const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; - internal const string DefaultValueAttributeType = "Microsoft.Azure.Functions.Worker.Annotations.DefaultValueAttribute"; + internal const string DefaultValueAttributeType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.DefaultValueAttribute"; // System types internal const string IEnumerableType = "System.Collections.IEnumerable";