diff --git a/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs new file mode 100644 index 000000000..189fb7027 --- /dev/null +++ b/extensions/Worker.Extensions.Abstractions/DefaultValueAttribute.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions +{ + [AttributeUsage(AttributeTargets.Property)] + public class DefaultValueAttribute : Attribute + { + /// + /// Define a default value for a property on a FunctionAttribute type. + /// + /// + public DefaultValueAttribute(string stringDefault) + { + DefaultStringValue = stringDefault; + } + + public DefaultValueAttribute(bool boolDefault) + { + DefaultBoolValue = boolDefault; + } + + public string? DefaultStringValue { get; } + + public bool? DefaultBoolValue { get; } + } +} diff --git a/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs b/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs new file mode 100644 index 000000000..15b4fce77 --- /dev/null +++ b/extensions/Worker.Extensions.Abstractions/ISupportCardinality.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Abstractions +{ + public interface ISupportCardinality + { + /// + /// Configures the "cardinality" property + /// This property indicates that the requested type is an array. Note that for + /// inputs and outputs, the effect of cardinality may be different (ex: electing to + /// receive a collection of events vs. indicating that my return type will be + /// a collection). + /// + public Cardinality Cardinality { get; set; } + } + + public enum Cardinality + { + Many, + One + } +} diff --git a/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs b/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs index d95cee778..7f7507509 100644 --- a/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs +++ b/extensions/Worker.Extensions.Abstractions/InputBindingAttribute.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System; diff --git a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs index f5d0ac59d..1b350eba9 100644 --- a/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/EventHubTriggerAttribute.cs @@ -5,8 +5,11 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class EventHubTriggerAttribute : TriggerBindingAttribute + public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + // Batch by default + private bool _isBatched = true; + /// /// Create an instance of this attribute. /// @@ -30,5 +33,41 @@ public EventHubTriggerAttribute(string eventHubName) /// Gets or sets the optional app setting name that contains the Event Hub connection string. If missing, tries to use a registered event hub receiver. /// public string? Connection { get; set; } + + /// + /// Gets or sets the configuration to enable batch processing of events. Default value is "true". + /// + [DefaultValue(true)] + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + + Cardinality ISupportCardinality.Cardinality + { + get + { + if (_isBatched) + { + return Cardinality.Many; + } + else + { + return Cardinality.One; + } + } + set + { + if (value.Equals(Cardinality.Many)) + { + _isBatched = true; + } + else + { + _isBatched = false; + } + } + } } } diff --git a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs index e75bc69b3..bd2d214cd 100644 --- a/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Kafka/KafkaTriggerAttribute.cs @@ -8,8 +8,10 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class KafkaTriggerAttribute : TriggerBindingAttribute + public sealed class KafkaTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + private bool _isBatched = false; + public KafkaTriggerAttribute(string brokerList, string topic) { BrokerList = brokerList; @@ -95,5 +97,39 @@ public KafkaTriggerAttribute(string brokerList, string topic) /// public string? SslKeyPassword { get; set; } + /// + /// Gets or sets the configuration to enable batch processing of events. Default value is "false". + /// + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + + Cardinality ISupportCardinality.Cardinality + { + get + { + if (_isBatched) + { + return Cardinality.Many; + } + else + { + return Cardinality.One; + } + } + set + { + if (value.Equals(Cardinality.Many)) + { + _isBatched = true; + } + else + { + _isBatched = false; + } + } + } } } diff --git a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs index 3e0580e6a..35ef6f461 100644 --- a/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs +++ b/extensions/Worker.Extensions.ServiceBus/ServiceBusTriggerAttribute.cs @@ -5,8 +5,10 @@ namespace Microsoft.Azure.Functions.Worker { - public sealed class ServiceBusTriggerAttribute : TriggerBindingAttribute + public sealed class ServiceBusTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { + private bool _isBatched = false; + private readonly string? _queueName; private readonly string? _topicName; private readonly string? _subscriptionName; @@ -67,5 +69,40 @@ public string? SubscriptionName /// Gets or sets a value indicating whether the sessions are enabled. /// public bool IsSessionsEnabled { get; set; } + + /// + /// Gets or sets the configuration to enable batch processing of events. Default value is "false". + /// + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + + Cardinality ISupportCardinality.Cardinality + { + get + { + if (_isBatched) + { + return Cardinality.Many; + } + else + { + return Cardinality.One; + } + } + set + { + if (value.Equals(Cardinality.Many)) + { + _isBatched = true; + } + else + { + _isBatched = false; + } + } + } } } diff --git a/release_notes.md b/release_notes.md index 9587e77cb..85979f321 100644 --- a/release_notes.md +++ b/release_notes.md @@ -6,3 +6,13 @@ - Updates to HttpResponseData - API updates - Support for response Cookies +- Add support for batched trigger events (#205) + - The following services allow trigger events to be batched: + - Event Hubs (batched by default) + - Service Bus (set `IsBatched = true` in trigger attribute) + - Kafka (set `IsBatched = true` in trigger attribute) + - To read batched event data in function code: + - Use array (`[]`), `IList`, `ICollection`, or `IEnumerable` if event data is `string`, `byte[]`, or `ReadOnlyMemory` (example: `string[]`). + - Note: `ReadOnlyMemory` is the more performant option to read binary data, especially for large payloads. + - Use a class that implements `IEnumerable` or `IEnumerable` for serializable event data (example: `List`). +- Fail function execution if the requested parameter cannot be converted to the specified type (#216) \ No newline at end of file diff --git a/sdk/Sdk/Constants.cs b/sdk/Sdk/Constants.cs new file mode 100644 index 000000000..1f3b6dd15 --- /dev/null +++ b/sdk/Sdk/Constants.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.Sdk +{ + internal static class Constants + { + // Our types + internal const string BindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.BindingAttribute"; + internal const string OutputBindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.OutputBindingAttribute"; + internal const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; + internal const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; + internal const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; + internal const string DefaultValueAttributeType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.DefaultValueAttribute"; + + // System types + internal const string IEnumerableType = "System.Collections.IEnumerable"; + internal const string IEnumerableGenericType = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfStringType = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfBinaryType = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfT = "System.Collections.Generic.IEnumerable`1"; + internal const string IEnumerableOfKeyValuePair = "System.Collections.Generic.IEnumerable`1>"; + internal const string GenericIEnumerableArgumentName = "T"; + internal const string StringType = "System.String"; + internal const string ByteArrayType = "System.Byte[]"; + internal const string LookupGenericType = "System.Linq.Lookup`2"; + internal const string DictionaryGenericType = "System.Collections.Generic.Dictionary`2"; + internal const string TaskGenericType = "System.Threading.Tasks.Task`1"; + internal const string TaskType = "System.Threading.Tasks.Task"; + internal const string VoidType = "System.Void"; + internal const string ReadOnlyMemoryOfBytes = "System.ReadOnlyMemory`1"; + + internal const string ReturnBindingName = "$return"; + internal const string HttpTriggerBindingType = "HttpTrigger"; + internal const string IsBatchedKey = "IsBatched"; + } +} diff --git a/sdk/Sdk/CustomAttributeExtensions.cs b/sdk/Sdk/CustomAttributeExtensions.cs index a1644a8fd..3f9f3d205 100644 --- a/sdk/Sdk/CustomAttributeExtensions.cs +++ b/sdk/Sdk/CustomAttributeExtensions.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System; @@ -14,16 +14,39 @@ public static class CustomAttributeExtensions public static IDictionary GetAllDefinedProperties(this CustomAttribute attribute) { var properties = new Dictionary(); - // To avoid needing to instantiate any types, assume that the constructor // argument names are equal to property names. + LoadDefaultProperties(properties, attribute); LoadConstructorArguments(properties, attribute); LoadDefinedProperties(properties, attribute); return properties; } - private static void LoadConstructorArguments(IDictionary properties, CustomAttribute attribute) + private static IEnumerable<(string, CustomAttributeArgument?)> GetDefaultValues(this CustomAttribute attribute) + { + return attribute.AttributeType.Resolve().Properties + .Select(p => (p.Name, p.CustomAttributes + .Where(attribute => string.Equals(attribute.AttributeType.FullName, Constants.DefaultValueAttributeType, StringComparison.Ordinal)) + .SingleOrDefault() + ?.ConstructorArguments.SingleOrDefault())) + .Where(t => t.Item2 is not null); + } + + private static void LoadDefaultProperties(IDictionary properties, CustomAttribute attribute) + { + var propertyDefaults = attribute.GetDefaultValues(); + + foreach (var propertyDefault in propertyDefaults) + { + if (propertyDefault.Item2 is not null) + { + properties[propertyDefault.Item1] = propertyDefault.Item2.Value.Value; + } + } + } + + private static void LoadConstructorArguments(IDictionary properties, CustomAttribute attribute) { var constructorParams = attribute.Constructor.Resolve().Parameters; for (int i = 0; i < attribute.ConstructorArguments.Count; i++) @@ -34,13 +57,12 @@ private static void LoadConstructorArguments(IDictionary propert string? paramName = param?.Name; object? paramValue = arg.Value; - if (paramName == null || paramValue == null) + if (paramName is null || paramValue is null) { continue; } - paramValue = GetEnrichedValue(param!.ParameterType, paramValue); - + paramValue = GetEnrichedValue(param!.ParameterType, paramValue); properties[paramName] = paramValue!; } } @@ -50,15 +72,16 @@ private static void LoadDefinedProperties(IDictionary properties foreach (CustomAttributeNamedArgument property in attribute.Properties) { object? propVal = property.Argument.Value; + string? propName = property.Name; - if (propVal == null) + if (propVal is null || propName is null) { continue; } propVal = GetEnrichedValue(property.Argument.Type, propVal); - properties[property.Name] = propVal!; + properties[propName] = propVal!; } } diff --git a/sdk/Sdk/DataTypeEnum.cs b/sdk/Sdk/DataTypeEnum.cs new file mode 100644 index 000000000..4a8925f74 --- /dev/null +++ b/sdk/Sdk/DataTypeEnum.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.Sdk +{ + internal enum DataType + { + Undefined, + Binary, + String, + Stream + } +} diff --git a/sdk/Sdk/FunctionMetadataGenerator.cs b/sdk/Sdk/FunctionMetadataGenerator.cs index e28dc3fdb..d74f80117 100644 --- a/sdk/Sdk/FunctionMetadataGenerator.cs +++ b/sdk/Sdk/FunctionMetadataGenerator.cs @@ -13,17 +13,6 @@ namespace Microsoft.Azure.Functions.Worker.Sdk { internal class FunctionMetadataGenerator { - private const string BindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.BindingAttribute"; - private const string OutputBindingType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.OutputBindingAttribute"; - private const string FunctionNameType = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; - private const string ExtensionsInformationType = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.ExtensionInformationAttribute"; - private const string HttpResponseType = "Microsoft.Azure.Functions.Worker.Http.HttpResponseData"; - private const string TaskGenericType = "System.Threading.Tasks.Task`1"; - private const string TaskType = "System.Threading.Tasks.Task"; - private const string VoidType = "System.Void"; - private const string ReturnBindingName = "$return"; - private const string HttpTriggerBindingType = "HttpTrigger"; - private readonly IndentableLogger _logger; // TODO: Verify that we don't need to allow @@ -90,6 +79,11 @@ public IEnumerable GenerateFunctionMetadata(string assembly { _logger.LogMessage($"Skipping file '{Path.GetFileName(path)}' because of a {nameof(BadImageFormatException)}."); } + catch (FunctionsMetadataGenerationException ex) + { + _logger.LogError($"Failed to generate function metadata from {Path.GetFileName(path)}."); + throw ex; + } catch (Exception ex) { _logger.LogWarning($"Could not evaluate '{Path.GetFileName(path)}' for functions metadata. Exception message: {ex.ToString()}"); @@ -153,7 +147,7 @@ private bool TryCreateFunctionMetadata(MethodDefinition method, out SdkFunctionM foreach (CustomAttribute attribute in method.CustomAttributes) { - if (string.Equals(attribute.AttributeType.FullName, FunctionNameType, StringComparison.Ordinal)) + if (string.Equals(attribute.AttributeType.FullName, Constants.FunctionNameType, StringComparison.Ordinal)) { string functionName = attribute.ConstructorArguments.SingleOrDefault().Value.ToString(); @@ -214,16 +208,16 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat { TypeReference? returnType = GetTaskElementType(method.ReturnType); - if (returnType is not null && !string.Equals(returnType.FullName, VoidType, StringComparison.Ordinal)) + if (returnType is not null && !string.Equals(returnType.FullName, Constants.VoidType, StringComparison.Ordinal)) { - if (string.Equals(returnType.FullName, HttpResponseType, StringComparison.Ordinal)) + if (string.Equals(returnType.FullName, Constants.HttpResponseType, StringComparison.Ordinal)) { - AddHttpOutputBinding(bindingMetadata, ReturnBindingName); + AddHttpOutputBinding(bindingMetadata, Constants.ReturnBindingName); } else { TypeDefinition returnDefinition = returnType.Resolve() - ?? throw new InvalidOperationException($"Couldn't find the type definition {returnType}"); + ?? throw new FunctionsMetadataGenerationException($"Couldn't find the type definition {returnType}"); bool hasOutputModel = TryAddOutputBindingsFromProperties(bindingMetadata, returnDefinition); @@ -232,7 +226,7 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat // support to other triggers without special handling if (!hasOutputModel && bindingMetadata.Any(d => IsHttpTrigger(d))) { - AddHttpOutputBinding(bindingMetadata, ReturnBindingName); + AddHttpOutputBinding(bindingMetadata, Constants.ReturnBindingName); } } } @@ -241,7 +235,7 @@ private void AddOutputBindingsFromReturnType(IList bindingMetadat private bool IsHttpTrigger(ExpandoObject bindingMetadata) { return bindingMetadata.Any(kvp => string.Equals(kvp.Key, "Type", StringComparison.Ordinal) - && string.Equals(kvp.Value?.ToString(), HttpTriggerBindingType, StringComparison.Ordinal)); + && string.Equals(kvp.Value?.ToString(), Constants.HttpTriggerBindingType, StringComparison.Ordinal)); } private bool TryAddOutputBindingsFromProperties(IList bindingMetadata, TypeDefinition typeDefinition) @@ -251,11 +245,11 @@ private bool TryAddOutputBindingsFromProperties(IList bindingMeta foreach (PropertyDefinition property in typeDefinition.Properties) { - if (string.Equals(property.PropertyType.FullName, HttpResponseType, StringComparison.Ordinal)) + if (string.Equals(property.PropertyType.FullName, Constants.HttpResponseType, StringComparison.Ordinal)) { if (foundHttpOutput) { - throw new InvalidOperationException($"Found multiple public properties with type '{HttpResponseType}' defined in output type '{typeDefinition.FullName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple public properties with type '{Constants.HttpResponseType}' defined in output type '{typeDefinition.FullName}'. " + $"Only one HTTP response binding type is supported in your return type definition."); } @@ -280,13 +274,13 @@ private void AddOutputBindingFromProperty(IList bindingMetadata, { if (foundOutputAttribute) { - throw new InvalidOperationException($"Found multiple output attributes on property '{property.Name}' defined in the function return type '{typeName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple output attributes on property '{property.Name}' defined in the function return type '{typeName}'. " + $"Only one output binding attribute is is supported on a property."); } foundOutputAttribute = true; - AddOutputBindingMetadata(bindingMetadata, propertyAttribute, property.Name); + AddOutputBindingMetadata(bindingMetadata, propertyAttribute, property.PropertyType, property.Name); AddExtensionInfo(_extensions, propertyAttribute); } } @@ -302,11 +296,11 @@ private bool TryAddOutputBindingFromMethod(IList bindingMetadata, { if (foundBinding) { - throw new Exception($"Found multiple Output bindings on method '{method.FullName}'. " + + throw new FunctionsMetadataGenerationException($"Found multiple Output bindings on method '{method.FullName}'. " + $"Please use an encapsulation to define the bindings in properties. For more information: https://aka.ms/dotnet-worker-poco-binding."); } - AddOutputBindingMetadata(bindingMetadata, methodAttribute, ReturnBindingName); + AddOutputBindingMetadata(bindingMetadata, methodAttribute, methodAttribute.AttributeType, Constants.ReturnBindingName); AddExtensionInfo(_extensions, methodAttribute); foundBinding = true; @@ -324,7 +318,7 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe { if (IsFunctionBindingType(parameterAttribute)) { - AddBindingMetadata(bindingMetadata, parameterAttribute, parameter.Name); + AddBindingMetadata(bindingMetadata, parameterAttribute, parameter.ParameterType, parameter.Name); AddExtensionInfo(_extensions, parameterAttribute); } } @@ -333,14 +327,14 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe private static TypeReference? GetTaskElementType(TypeReference typeReference) { - if (typeReference is null || typeReference.FullName == TaskType) + if (typeReference is null || string.Equals(typeReference.FullName, Constants.TaskType, StringComparison.Ordinal)) { return null; } if (typeReference.IsGenericInstance && typeReference is GenericInstanceType genericType - && string.Equals(typeReference.GetElementType().FullName, TaskGenericType, StringComparison.Ordinal)) + && string.Equals(typeReference.GetElementType().FullName, Constants.TaskGenericType, StringComparison.Ordinal)) { // T from Task return genericType.GenericArguments[0]; @@ -351,20 +345,20 @@ private void AddInputTriggerBindingsAndExtensions(IList bindingMe } } - private static void AddOutputBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? name = null) + private static void AddOutputBindingMetadata(IList bindingMetadata, CustomAttribute attribute, TypeReference parameterType, string? name = null) { - AddBindingMetadata(bindingMetadata, attribute, parameterName: name); + AddBindingMetadata(bindingMetadata, attribute, parameterType, parameterName: name); } - private static void AddBindingMetadata(IList bindingMetadata, CustomAttribute attribute, string? parameterName) + private static void AddBindingMetadata(IList bindingMetadata, CustomAttribute attribute, TypeReference parameterType, string? parameterName) { string bindingType = GetBindingType(attribute); - ExpandoObject binding = BuildBindingMetadataFromAttribute(attribute, bindingType, parameterName); + ExpandoObject binding = BuildBindingMetadataFromAttribute(attribute, bindingType, parameterType, parameterName); bindingMetadata.Add(binding); } - private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute attribute, string bindingType, string? parameterName) + private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute attribute, string bindingType, TypeReference parameterType, string? parameterName) { ExpandoObject binding = new ExpandoObject(); @@ -378,14 +372,228 @@ private static ExpandoObject BuildBindingMetadataFromAttribute(CustomAttribute a bindingDict["Type"] = bindingType; bindingDict["Direction"] = GetBindingDirection(attribute); + // Is string parameter type + if (IsStringType(parameterType.FullName)) + { + bindingDict["DataType"] = "String"; + } + // Is binary parameter type + else if (IsBinaryType(parameterType.FullName)) + { + bindingDict["DataType"] = "Binary"; + } + foreach (var property in attribute.GetAllDefinedProperties()) { bindingDict.Add(property.Key, property.Value); } + // Determine if we should set the "Cardinality" property based on + // the presence of "IsBatched." This is a property that is from the + // attributes that implement the ISupportCardinality interface. + // + // Note that we are directly looking for "IsBatched" today while we + // are not actually instantiating the Attribute type and instead relying + // on type inspection via Mono.Cecil. + // TODO: Do not hard-code "IsBatched" as the property to set cardinality. + // We should rely on the interface + // + // Conversion rule + // "IsBatched": true => "Cardinality": "Many" + // "IsBatched": false => "Cardinality": "One" + if (bindingDict.TryGetValue(Constants.IsBatchedKey, out object isBatchedValue) + && isBatchedValue is bool isBatched) + { + // Batching set to true + if (isBatched) + { + bindingDict["Cardinality"] = "Many"; + // Throw if parameter type is *definitely* not a collection type. + // Note that this logic doesn't dictate what we can/can't do, and + // we can be more restrictive in the future because today some + // scenarios result in runtime failures. + if (IsIterableCollection(parameterType, out DataType dataType)) + { + if (dataType.Equals(DataType.String)) + { + bindingDict["DataType"] = "String"; + } + else if (dataType.Equals(DataType.Binary)) + { + bindingDict["DataType"] = "Binary"; + } + } + else + { + throw new FunctionsMetadataGenerationException("Function is configured to process events in batches but parameter type is not iterable. " + + $"Change parameter named '{ parameterName }' to be an IEnumerable type or set 'IsBatched' to false on your '{attribute.AttributeType.Name.Replace("Attribute", "")}' attribute."); + } + } + // Batching set to false + else + { + bindingDict["Cardinality"] = "One"; + } + + bindingDict.Remove(Constants.IsBatchedKey); + } + return binding; } + private static bool IsIterableCollection(TypeReference type, out DataType dataType) + { + // Array and not byte array + bool isArray = type.IsArray && !string.Equals(type.FullName, Constants.ByteArrayType, StringComparison.Ordinal); + if (isArray) + { + TypeSpecification? typeSpecification = type as TypeSpecification; + if (typeSpecification is not null) + { + dataType = GetDataTypeFromType(typeSpecification.ElementType.FullName); + return true; + } + } + + bool isMappingEnumerable = IsOrDerivedFrom(type, Constants.IEnumerableOfKeyValuePair) + || IsOrDerivedFrom(type, Constants.LookupGenericType) + || IsOrDerivedFrom(type, Constants.DictionaryGenericType); + if (isMappingEnumerable) + { + dataType = DataType.Undefined; + return false; + } + + // IEnumerable and not string or dictionary + bool isEnumerableOfT = IsOrDerivedFrom(type, Constants.IEnumerableOfT); + bool isEnumerableCollection = + !IsStringType(type.FullName) + && (IsOrDerivedFrom(type, Constants.IEnumerableType) + || IsOrDerivedFrom(type, Constants.IEnumerableGenericType) + || isEnumerableOfT); + if (isEnumerableCollection) + { + dataType = DataType.Undefined; + if (IsOrDerivedFrom(type, Constants.IEnumerableOfStringType)) + { + dataType = DataType.String; + } + else if (IsOrDerivedFrom(type, Constants.IEnumerableOfBinaryType)) + { + dataType = DataType.Binary; + } + else if (isEnumerableOfT) + { + // Find real type that "T" in IEnumerable resolves to + string typeName = ResolveIEnumerableOfTType(type, new Dictionary()) ?? string.Empty; + dataType = GetDataTypeFromType(typeName); + } + return true; + } + + dataType = DataType.Undefined; + return false; + } + + private static bool IsOrDerivedFrom(TypeReference type, string interfaceFullName) + { + bool isType = string.Equals(type.FullName, interfaceFullName, StringComparison.Ordinal); + TypeDefinition definition = type.Resolve(); + return isType || IsDerivedFrom(definition, interfaceFullName); + } + + private static bool IsDerivedFrom(TypeDefinition definition, string interfaceFullName) + { + var isType = string.Equals(definition.FullName, interfaceFullName, StringComparison.Ordinal); + return isType || HasInterface(definition, interfaceFullName) || IsSubclassOf(definition, interfaceFullName); + } + + private static bool HasInterface(TypeDefinition definition, string interfaceFullName) + { + return definition.Interfaces.Any(i => string.Equals(i.InterfaceType.FullName, interfaceFullName, StringComparison.Ordinal)); + } + + private static bool IsSubclassOf(TypeDefinition definition, string interfaceFullName) + { + if (definition.BaseType is null) + { + return false; + } + + TypeDefinition baseType = definition.BaseType.Resolve(); + return IsDerivedFrom(baseType, interfaceFullName); + } + + private static string? ResolveIEnumerableOfTType(TypeReference type, Dictionary foundMapping) + { + // Base case: + // We are at IEnumerable and want to return the most recent resolution of T + // (Most recent is relative to IEnumerable) + if (string.Equals(type.FullName, Constants.IEnumerableOfT, StringComparison.Ordinal)) + { + if (foundMapping.TryGetValue(Constants.GenericIEnumerableArgumentName, out string typeName)) + { + return typeName; + } + else + { + return null; + } + } + + TypeDefinition definition = type.Resolve(); + if (definition.HasGenericParameters && type is GenericInstanceType genericType) + { + for (int i = 0; i < genericType.GenericArguments.Count(); i++) + { + string name = genericType.GenericArguments.ElementAt(i).FullName; + string resolvedName = definition.GenericParameters.ElementAt(i).FullName; + + if (foundMapping.TryGetValue(name, out string firstType)) + { + foundMapping.Remove(name); + foundMapping.Add(resolvedName, firstType); + } + else + { + foundMapping.Add(resolvedName, name); + } + } + + } + + return definition.Interfaces + .Select(i => ResolveIEnumerableOfTType(i.InterfaceType, foundMapping)) + .Where(name => name is not null) + .FirstOrDefault() + ?? ResolveIEnumerableOfTType(definition.BaseType, foundMapping); + } + + private static DataType GetDataTypeFromType(string fullName) + { + if (IsStringType(fullName)) + { + return DataType.String; + } + else if (IsBinaryType(fullName)) + { + return DataType.Binary; + } + + return DataType.Undefined; + } + + private static bool IsStringType(string fullName) + { + return string.Equals(fullName, Constants.StringType, StringComparison.Ordinal); + } + + private static bool IsBinaryType(string fullName) + { + return string.Equals(fullName, Constants.ByteArrayType, StringComparison.Ordinal) + || string.Equals(fullName, Constants.ReadOnlyMemoryOfBytes, StringComparison.Ordinal); + } + private static string GetBindingType(CustomAttribute attribute) { var attributeType = attribute.AttributeType.Name; @@ -413,7 +621,7 @@ private static void AddExtensionInfo(IDictionary extensions, Cus foreach (var assemblyAttribute in extensionAssemblyDefintion.CustomAttributes) { - if (string.Equals(assemblyAttribute.AttributeType.FullName, ExtensionsInformationType, StringComparison.Ordinal)) + if (string.Equals(assemblyAttribute.AttributeType.FullName, Constants.ExtensionsInformationType, StringComparison.Ordinal)) { string extensionName = assemblyAttribute.ConstructorArguments[0].Value.ToString(); string extensionVersion = assemblyAttribute.ConstructorArguments[1].Value.ToString(); @@ -438,12 +646,12 @@ private static string GetBindingDirection(CustomAttribute attribute) private static bool IsOutputBindingType(CustomAttribute attribute) { - return TryGetBaseAttributeType(attribute, OutputBindingType, out _); + return TryGetBaseAttributeType(attribute, Constants.OutputBindingType, out _); } private static bool IsFunctionBindingType(CustomAttribute attribute) { - return TryGetBaseAttributeType(attribute, BindingType, out _); + return TryGetBaseAttributeType(attribute, Constants.BindingType, out _); } private static bool TryGetBaseAttributeType(CustomAttribute attribute, string baseType, out TypeReference? baseTypeRef) diff --git a/sdk/Sdk/FunctionsMetadataGenerationException.cs b/sdk/Sdk/FunctionsMetadataGenerationException.cs new file mode 100644 index 000000000..1aa12f862 --- /dev/null +++ b/sdk/Sdk/FunctionsMetadataGenerationException.cs @@ -0,0 +1,12 @@ +using System; + +namespace Microsoft.Azure.Functions.Worker.Sdk +{ + internal class FunctionsMetadataGenerationException: Exception + { + internal FunctionsMetadataGenerationException(string message): base(message) { } + + internal FunctionsMetadataGenerationException(string message, Exception innerException) : base(message, innerException) { } + + } +} diff --git a/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs b/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs index 837c92485..b687bfbf3 100644 --- a/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs +++ b/src/DotNetWorker/Context/Features/GrpcFunctionBindingsFeature.cs @@ -116,6 +116,12 @@ public void SetOutputBinding(string name, object value) // This is guaranteed to be Json here -- we can use that. TypedData.DataOneofCase.Json => typedData.Json, TypedData.DataOneofCase.Bytes => typedData.Bytes.Memory, + TypedData.DataOneofCase.CollectionBytes => typedData.CollectionBytes.Bytes.Select(element => { + return element.Memory; + }), + TypedData.DataOneofCase.CollectionString => typedData.CollectionString.String, + TypedData.DataOneofCase.CollectionDouble => typedData.CollectionDouble.Double, + TypedData.DataOneofCase.CollectionSint64 => typedData.CollectionSint64.Sint64, _ => throw new NotSupportedException($"{typedData.DataCase} is not supported."), }; } diff --git a/src/DotNetWorker/Converters/ArrayConverter.cs b/src/DotNetWorker/Converters/ArrayConverter.cs new file mode 100644 index 000000000..a876563e5 --- /dev/null +++ b/src/DotNetWorker/Converters/ArrayConverter.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Azure.Functions.Worker.Converters +{ + // Converting IEnumerable<> to Array + internal class ArrayConverter : IConverter + { + // Convert IEnumerable to array + public bool TryConvert(ConverterContext context, out object? target) + { + target = null; + // Ensure requested type is an array + if (context.Parameter.Type.IsArray) + { + Type? elementType = context.Parameter.Type.GetElementType(); + if (elementType is not null) + { + // Ensure that we can assign from source to parameter type + if (elementType.Equals(typeof(string)) + || elementType.Equals(typeof(byte[])) + || elementType.Equals(typeof(ReadOnlyMemory)) + || elementType.Equals(typeof(long)) + || elementType.Equals(typeof(double))) + { + target = context.Source switch + { + IEnumerable source => source.ToArray(), + IEnumerable> source => GetBinaryData(source, elementType!), + IEnumerable source => source.ToArray(), + IEnumerable source => source.ToArray(), + _ => null + }; + } + } + } + + return target is not null; + } + + private static object? GetBinaryData(IEnumerable> source, Type targetType) + { + if (targetType.IsAssignableFrom(typeof(ReadOnlyMemory))) + { + return source.ToArray(); + } + else + { + return source.Select(i => i.ToArray()).ToArray(); + } + } + } +} diff --git a/src/DotNetWorker/GrpcWorker.cs b/src/DotNetWorker/GrpcWorker.cs index eb30b1bb9..fe2bcc886 100644 --- a/src/DotNetWorker/GrpcWorker.cs +++ b/src/DotNetWorker/GrpcWorker.cs @@ -207,6 +207,7 @@ internal async Task WorkerInitRequestHandlerAsync(StreamingMessage request) response.Capabilities.Add("RawHttpBodyBytes", bool.TrueString); response.Capabilities.Add("RpcHttpTriggerMetadataRemoved", bool.TrueString); response.Capabilities.Add("UseNullableValueDictionaryForHttp", bool.TrueString); + response.Capabilities.Add("TypedDataCollection", bool.TrueString); StreamingMessage responseMessage = new StreamingMessage { diff --git a/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs b/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs index 2aa25778a..9f8698495 100644 --- a/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs +++ b/src/DotNetWorker/Hosting/ServiceCollectionExtensions.cs @@ -109,7 +109,8 @@ internal static IServiceCollection RegisterDefaultConverters(this IServiceCollec .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); } internal static IServiceCollection RegisterOutputChannel(this IServiceCollection services) diff --git a/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs b/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs new file mode 100644 index 000000000..d6b2be5be --- /dev/null +++ b/test/DotNetWorkerTests/Converters/ArrayConverterTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Text; +using Google.Protobuf.Collections; +using Microsoft.Azure.Functions.Worker.Converters; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Tests.Converters +{ + public class ArrayConverterTests + { + private const string _sourceString = "hello"; + private static readonly byte[] _sourceBytes = Encoding.UTF8.GetBytes(_sourceString); + private static readonly ReadOnlyMemory _sourceMemory = new ReadOnlyMemory(_sourceBytes); + private ArrayConverter _converter = new ArrayConverter(); + + private static readonly IEnumerable> _sourceMemoryEnumerable = new RepeatedField>() { _sourceMemory }; + private static readonly RepeatedField _sourceStringEnumerable = new RepeatedField() { _sourceString }; + private static readonly RepeatedField _sourceDoubleEnumerable = new RepeatedField() { 1.0 }; + private static readonly RepeatedField _sourceLongEnumerable = new RepeatedField() { 2000 }; + + [Fact] + public void ConvertCollectionBytesToJaggedByteArray() + { + var context = new TestConverterContext("output", typeof(byte[][]), _sourceMemoryEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + + [Fact] + public void ConvertCollectionBytesToReadOnlyByteArray() + { + var context = new TestConverterContext("output", typeof(ReadOnlyMemory[]), _sourceMemoryEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert[]>(target); + } + + [Fact] + public void ConvertCollectionStringToStringArray() + { + var context = new TestConverterContext("output", typeof(string[]), _sourceStringEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + + [Fact] + public void ConvertCollectionSint64ToLongArray() + { + var context = new TestConverterContext("output", typeof(long[]), _sourceLongEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + + [Fact] + public void ConvertCollectionDoubleToDoubleArray() + { + var context = new TestConverterContext("output", typeof(double[]), _sourceDoubleEnumerable); + Assert.True(_converter.TryConvert(context, out object target)); + TestUtility.AssertIsTypeAndConvert(target); + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj index 245e53bdb..1f11d3b95 100644 --- a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj +++ b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj @@ -14,6 +14,7 @@ + diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs new file mode 100644 index 000000000..1e3b35030 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsEnumerableFunctions.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using Microsoft.Azure.Functions.Worker.E2EApp.EventHubs; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp +{ + public static class EventHubsEnumerableFunctions + { + [Function(nameof(EventHubsEnumerableTrigger))] + [EventHubOutput("test-output-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData EventHubsEnumerableTrigger([EventHubTrigger("test-input-enumerable-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsEnumerableTrigger)); + logger.LogInformation($"First trigger (List)!!"); + input.ForEach(item => logger.LogInformation(item.ToString())); + return input[0]; + } + + [Function(nameof(EventHubsVerifyOutputEnumerable))] + [QueueOutput("test-eventhub-output-string-dotnet-isolated")] + public static string EventHubsVerifyOutputEnumerable([EventHubTrigger("test-output-enumerable-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsVerifyOutputEnumerable)); + logger.LogInformation($"Second trigger (List)!! '{input[0]}'"); + return input[0]; + } + + [Function(nameof(TestEnumerable))] + [EventHubOutput("test-input-enumerable-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData TestEnumerable( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestData req, + FunctionContext context) + { + var logger = context.GetLogger(nameof(TestEnumerable)); + logger.LogInformation(".NET Worker HTTP trigger function processed a request"); + return new TestData() + { + Name = "Ballmer", + TimeProperty = "2021-01-27T15:57:38.000-06:00" + }; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs new file mode 100644 index 000000000..44850db93 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsObjectFunctions.cs @@ -0,0 +1,43 @@ +using Microsoft.Azure.Functions.Worker.E2EApp.EventHubs; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp +{ + public static class EventHubsObjectFunctions + { + [Function(nameof(EventHubsObjectFunction))] + [EventHubOutput("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData EventHubsObjectFunction([EventHubTrigger("test-eventhub-input-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting", IsBatched = false)] TestData input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsObjectFunction)); + logger.LogInformation($"First Trigger (TestData)!! Name: '{input.Name}' Time: '{input.TimeProperty}'"); + return input; + } + + [Function(nameof(EventHubsVerifyOutputObject))] + [QueueOutput("test-eventhub-output-object-dotnet-isolated")] + public static TestData EventHubsVerifyOutputObject([EventHubTrigger("test-eventhub-output-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting", IsBatched = false)] TestData input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsVerifyOutputObject)); + logger.LogInformation($"Second Trigger (TestData)!! Name: '{input.Name}' Time: '{input.TimeProperty}'"); + return input; + } + + [Function(nameof(TestObject))] + [EventHubOutput("test-eventhub-input-object-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static TestData TestObject( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestData req, + FunctionContext context) + { + var logger = context.GetLogger(nameof(TestObject)); + return new TestData() + { + Name = "Ballmer", + TimeProperty = "2021-01-27T15:57:38.000-09:00" + }; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs new file mode 100644 index 000000000..09c2143fb --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/EventHubsStringFunctions.cs @@ -0,0 +1,39 @@ +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp +{ + public static class EventHubsStringFunctions + { + [Function(nameof(EventHubsStringTrigger))] + [EventHubOutput("test-output-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static string EventHubsStringTrigger([EventHubTrigger("test-input-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsStringTrigger)); + logger.LogInformation($"First trigger (string[])!! '{input[0]}'"); + return input[0]; + } + + [Function(nameof(EventHubsVerifyOutputString))] + [QueueOutput("test-eventhub-output-string-dotnet-isolated")] + public static string EventHubsVerifyOutputString([EventHubTrigger("test-output-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + var logger = context.GetLogger(nameof(EventHubsVerifyOutputString)); + logger.LogInformation($"Second trigger (string[])!! '{input[0]}'"); + return input[0]; + } + + [Function(nameof(Test))] + [EventHubOutput("test-input-string-dotnet-isolated", Connection = "EventHubConnectionAppSetting")] + public static string Test( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestData req, + FunctionContext context) + { + var logger = context.GetLogger(nameof(Test)); + logger.LogInformation(".NET Worker HTTP trigger function processed a request"); + return "hello world"; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs b/test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs new file mode 100644 index 000000000..3daa270b4 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/EventHubs/TestData.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Functions.Worker.E2EApp.EventHubs +{ + public class TestData + { + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("TimeProperty")] + public string TimeProperty { get; set; } + + public override string ToString() + { + return $"Name: {Name}, TimeProperty: {TimeProperty}"; + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs index 4ab45a68f..aee020dd0 100644 --- a/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/Http/BasicHttpFunctions.cs @@ -2,12 +2,10 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System.Net; -using System.Text; using System.Text.Json; using System.Web; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; -using Microsoft.Azure.Functions.Worker.Pipeline; using Microsoft.Extensions.Logging; namespace Microsoft.Azure.Functions.Worker.E2EApp diff --git a/test/E2ETests/E2EApps/E2EApp/local.settings.json b/test/E2ETests/E2EApps/E2EApp/local.settings.json index 7e1a57612..074751b6b 100644 --- a/test/E2ETests/E2EApps/E2EApp/local.settings.json +++ b/test/E2ETests/E2EApps/E2EApp/local.settings.json @@ -6,6 +6,7 @@ "CosmosConnection": "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "CosmosDb": "ItemDb", "CosmosCollIn": "ItemCollectionIn", - "CosmosCollOut": "ItemCollectionOut" + "CosmosCollOut": "ItemCollectionOut", + "EventHubConnectionAppSetting": "%EventHubConnectionAppSetting%" } } \ No newline at end of file diff --git a/test/E2ETests/E2ETests/E2ETests.sln b/test/E2ETests/E2ETests/E2ETests.sln index 55443eaa7..1a3e1ac90 100644 --- a/test/E2ETests/E2ETests/E2ETests.sln +++ b/test/E2ETests/E2ETests/E2ETests.sln @@ -23,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.CosmosDB" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.Storage", "..\..\..\extensions\Worker.Extensions.Storage\Worker.Extensions.Storage.csproj", "{34894B4F-B58C-49E8-84D7-D6AACA200B06}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.EventHubs", "..\..\..\extensions\Worker.Extensions.EventHubs\Worker.Extensions.EventHubs.csproj", "{09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -61,6 +63,10 @@ Global {34894B4F-B58C-49E8-84D7-D6AACA200B06}.Debug|Any CPU.Build.0 = Debug|Any CPU {34894B4F-B58C-49E8-84D7-D6AACA200B06}.Release|Any CPU.ActiveCfg = Release|Any CPU {34894B4F-B58C-49E8-84D7-D6AACA200B06}.Release|Any CPU.Build.0 = Release|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {09358C05-8F2F-49A6-B6D1-DDDB8BB3D3E9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 82ffe81f8..1d2aa5aac 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Dynamic; using System.Linq; @@ -109,7 +111,7 @@ public void BasicHttpFunctionWithExternalReturnType() }); ValidateFunction(functions.Single(), ExternalType_Return.FunctionName, GetEntryPoint(nameof(ExternalType_Return), nameof(ExternalType_Return.Http)), - b => ValidateTrigger(b), + b => ValidateTrigger(b), b => ValidateQueueOutput(b)); void ValidateTrigger(ExpandoObject b) @@ -169,7 +171,8 @@ void ValidateQueueTrigger(ExpandoObject b) { "Type", "QueueTrigger" }, { "Direction", "In" }, { "Connection", "MyConnection" }, - { "queueName", "queueName" } + { "queueName", "queueName" }, + { "DataType", "String" } }); } @@ -196,7 +199,8 @@ void ValidateBlobTrigger(ExpandoObject b) { "Name", "blob" }, { "Type", "BlobTrigger" }, { "Direction", "In" }, - { "blobPath", "container2/%file%" } + { "blobPath", "container2/%file%" }, + { "DataType", "String" } }); } @@ -270,7 +274,8 @@ void ValidateQueueTrigger(ExpandoObject b) { "Type", "QueueTrigger" }, { "Direction", "In" }, { "Connection", "MyConnection" }, - { "queueName", "queueName" } + { "queueName", "queueName" }, + { "DataType", "String" } }); } @@ -282,7 +287,8 @@ void ValidateBlobOutput(ExpandoObject b) { "Type", "Blob" }, { "Direction", "Out" }, { "blobPath", "container1/hello.txt" }, - { "Connection", "MyOtherConnection" } + { "Connection", "MyOtherConnection" }, + { "DataType", "String" } }); } @@ -294,6 +300,7 @@ void ValidateQueueOutput(ExpandoObject b) { "Type", "Queue" }, { "Direction", "Out" }, { "queueName", "queue2" }, + { "DataType", "String" } }); } } @@ -328,7 +335,8 @@ void ValidateHttpTrigger(ExpandoObject b) { "Name", "req" }, { "Type", "HttpTrigger" }, { "Direction", "In" }, - { "methods", new[] { "get" } } + { "methods", new[] { "get" } }, + { "DataType", "String" } }); } @@ -350,6 +358,7 @@ void ValidateQueueOutput(ExpandoObject b) { "Type", "Queue" }, { "Direction", "Out" }, { "queueName", "queue2" }, + { "DataType", "String" } }); } } @@ -380,7 +389,8 @@ void ValidateHttpTrigger(ExpandoObject b) { "Name", "req" }, { "Type", "HttpTrigger" }, { "Direction", "In" }, - { "methods", new[] { "get" } } + { "methods", new[] { "get" } }, + { "DataType", "String" } }); } @@ -402,11 +412,104 @@ public void MultiOutput_OnMethod_Throws() var module = ModuleDefinition.ReadModule(_thisAssembly.Location); var typeDef = TestUtility.GetTypeDefinition(typeof(MultiOutput_Method)); - var exception = Assert.Throws(() => generator.GenerateFunctionMetadata(typeDef)); + var exception = Assert.Throws(() => generator.GenerateFunctionMetadata(typeDef)); Assert.Contains($"Found multiple Output bindings on method", exception.Message); } + [Theory] + [InlineData("StringInputFunction", nameof(CardinalityMany.StringInputFunction), false, "String")] + [InlineData("StringArrayInputFunction", nameof(CardinalityMany.StringArrayInputFunction), true, "String")] + [InlineData("BinaryInputFunction", nameof(CardinalityMany.BinaryInputFunction), false, "Binary")] + [InlineData("BinaryArrayInputFunction", nameof(CardinalityMany.BinaryArrayInputFunction), true, "Binary")] + [InlineData("IntArrayInputFunction", nameof(CardinalityMany.IntArrayInputFunction), true, "")] + [InlineData("StringListInputFunction", nameof(CardinalityMany.StringListInputFunction), true, "String")] + [InlineData("BinaryListInputFunction", nameof(CardinalityMany.BinaryListInputFunction), true, "Binary")] + [InlineData("EnumerableNestedStringGenericClassInputFunction", nameof(CardinalityMany.EnumerableNestedStringGenericClassInputFunction), true, "String")] + [InlineData("EnumerableNestedStringGenericClass2InputFunction", nameof(CardinalityMany.EnumerableNestedStringGenericClass2InputFunction), true, "String")] + [InlineData("IntListInputFunction", nameof(CardinalityMany.IntListInputFunction), true, "")] + [InlineData("StringDoubleArrayInputFunction", nameof(CardinalityMany.StringDoubleArrayInputFunction), true, "")] + [InlineData("EnumerableClassInputFunction", nameof(CardinalityMany.EnumerableClassInputFunction), true, "")] + [InlineData("EnumerableStringClassInputFunction", nameof(CardinalityMany.EnumerableStringClassInputFunction), true, "String")] + [InlineData("EnumerableBinaryClassInputFunction", nameof(CardinalityMany.EnumerableBinaryClassInputFunction), true, "Binary")] + [InlineData("EnumerableGenericClassInputFunction", nameof(CardinalityMany.EnumerableGenericClassInputFunction), true, "")] + [InlineData("EnumerableNestedBinaryClassInputFunction", nameof(CardinalityMany.EnumerableNestedBinaryClassInputFunction), true, "Binary")] + [InlineData("EnumerableNestedStringClassInputFunction", nameof(CardinalityMany.EnumerableNestedStringClassInputFunction), true, "String")] + [InlineData("LookupInputFunction", nameof(CardinalityMany.LookupInputFunction), false, "")] + [InlineData("DictionaryInputFunction", nameof(CardinalityMany.DictionaryInputFunction), false, "")] + [InlineData("ConcurrentDictionaryInputFunction", nameof(CardinalityMany.ConcurrentDictionaryInputFunction), false, "")] + [InlineData("HashSetInputFunction", nameof(CardinalityMany.HashSetInputFunction), true, "String")] + [InlineData("EnumerableInputFunction", nameof(CardinalityMany.EnumerableInputFunction), true, "")] + [InlineData("EnumerableStringInputFunction", nameof(CardinalityMany.EnumerableStringInputFunction), true, "String")] + [InlineData("EnumerableBinaryInputFunction", nameof(CardinalityMany.EnumerableBinaryInputFunction), true, "Binary")] + [InlineData("EnumerableGenericInputFunction", nameof(CardinalityMany.EnumerableGenericInputFunction), true, "")] + [InlineData("EnumerablePocoInputFunction", nameof(CardinalityMany.EnumerablePocoInputFunction), true, "")] + [InlineData("ListPocoInputFunction", nameof(CardinalityMany.ListPocoInputFunction), true, "")] + public void CardinalityManyFunctions(string functionName, string entryPoint, bool cardinalityMany, string dataType) + { + var generator = new FunctionMetadataGenerator(); + var typeDef = TestUtility.GetTypeDefinition(typeof(CardinalityMany)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + SdkFunctionMetadata metadata = functions.Where(a => string.Equals(a.Name, functionName, StringComparison.Ordinal)).Single(); + + ValidateFunction(metadata, functionName, GetEntryPoint(nameof(CardinalityMany), entryPoint), + b => ValidateTrigger(b, cardinalityMany)); + + AssertDictionary(extensions, new Dictionary(){ + { "Microsoft.Azure.WebJobs.Extensions.EventHubs", "4.2.0" } + }); + + void ValidateTrigger(ExpandoObject b, bool many) + { + var expected = new Dictionary() + { + { "Name", "input" }, + { "Type", "EventHubTrigger" }, + { "Direction", "In" }, + { "eventHubName", "test" }, + { "Connection", "EventHubConnectionAppSetting" } + }; + + if (many) + { + expected.Add("Cardinality", "Many"); + } + else + { + expected.Add("Cardinality", "One"); + } + + if (!string.IsNullOrEmpty(dataType)) + { + expected.Add("DataType", dataType); + } + + AssertExpandoObject(b, expected); + } + } + + [Fact] + public void CardinalityMany_WithNotIterableTypeThrows() + { + var generator = new FunctionMetadataGenerator(); + var typeDef = TestUtility.GetTypeDefinition(typeof(EventHubNotBatched)); + + var exception = Assert.Throws(() => generator.GenerateFunctionMetadata(typeDef)); + Assert.Contains("Function is configured to process events in batches but parameter type is not iterable", exception.Message); + } + + private class EventHubNotBatched + { + [Function("EventHubTrigger")] + public static void EventHubTrigger([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string input, + FunctionContext context) + { + throw new NotImplementedException(); + } + } + private static string GetEntryPoint(string className, string methodName) => $"{typeof(FunctionMetadataGeneratorTests).FullName}+{className}.{methodName}"; private void ValidateFunction(SdkFunctionMetadata sdkFunctionMetadata, string name, string entryPoint, params Action[] bindingValidations) @@ -429,7 +532,7 @@ private static void AssertExpandoObject(ExpandoObject expando, IDictionary(IDictionary dict, IDictionary expected) + private static void AssertDictionary(IDictionary dict, IDictionary expected) { Assert.Equal(expected.Count, dict.Count); @@ -568,5 +671,270 @@ public Task RunTimer([TimerTrigger("0 0 0 * * *", RunOnStartup = false)] object throw new NotImplementedException(); } } + + private class CardinalityMany + { + [Function("StringInputFunction")] + public static void StringInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] string input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("StringArrayInputFunction")] + public static void StringArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("BinaryInputFunction")] + public static void BinaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] byte[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("BinaryArrayInputFunction")] + public static void BinaryArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] byte[][] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("IntArrayInputFunction")] + public static void IntArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] int[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("StringListInputFunction")] + public static void StringListInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("StringDoubleArrayInputFunction")] + public static void StringDoubleArrayInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] string[][] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("BinaryListInputFunction")] + public static void BinaryListInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("IntListInputFunction")] + public static void IntListInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] int[] input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableClassInputFunction")] + public static void EnumerableClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableStringClassInputFunction")] + public static void EnumerableStringClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableBinaryClassInputFunction")] + public static void EnumerableBinaryClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableBinaryTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableGenericClassInputFunction")] + public static void EnumerableGenericClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableGenericTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedBinaryClassInputFunction")] + public static void EnumerableNestedBinaryClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableBinaryNestedTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedStringClassInputFunction")] + public static void EnumerableNestedStringClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringNestedTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedStringGenericClassInputFunction")] + public static void EnumerableNestedStringGenericClassInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringNestedGenericTestClass input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableNestedStringGenericClass2InputFunction")] + public static void EnumerableNestedStringGenericClass2InputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] EnumerableStringNestedGenericTestClass2 input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("LookupInputFunction")] + public static void LookupInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] Lookup input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("DictionaryInputFunction")] + public static void DictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] Dictionary input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("ConcurrentDictionaryInputFunction")] + public static void ConcurrentDictionaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting", IsBatched = false)] ConcurrentDictionary input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("HashSetInputFunction")] + public static void HashSetInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] HashSet input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableInputFunction")] + public static void EnumerableInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableStringInputFunction")] + public static void EnumerableStringInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableBinaryInputFunction")] + public static void EnumerableBinaryInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerableGenericInputFunction")] + public static void EnumerableGenericInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("EnumerablePocoInputFunction")] + public static void EnumerablePocoInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] IEnumerable input, + FunctionContext context) + { + throw new NotImplementedException(); + } + + [Function("ListPocoInputFunction")] + public static void ListPocoInputFunction([EventHubTrigger("test", Connection = "EventHubConnectionAppSetting")] List input, + FunctionContext context) + { + throw new NotImplementedException(); + } + } + + private class EnumerableTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class EnumerableStringTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class EnumerableBinaryTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class EnumerableStringTestClass : List + { + } + + private class EnumerableBinaryNestedTestClass : EnumerableBinaryTestClass + { + } + + private class EnumerableStringNestedTestClass : EnumerableStringTestClass + { + } + + private class EnumerableStringNestedGenericTestClass2 : EnumerableStringNestedGenericTestClass + { + } + + private class EnumerableStringNestedGenericTestClass : EnumerableStringTestClass + { + } + + private class EnumerableGenericTestClass : IEnumerable + { + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + private class Poco + { + public string Foo; + public string Bar; + } } } diff --git a/test/FunctionMetadataGeneratorTests/SdkTests.csproj b/test/FunctionMetadataGeneratorTests/SdkTests.csproj index 34ab4a2f1..2275a2f5f 100644 --- a/test/FunctionMetadataGeneratorTests/SdkTests.csproj +++ b/test/FunctionMetadataGeneratorTests/SdkTests.csproj @@ -28,6 +28,7 @@ + diff --git a/test/SdkE2ETests/Contents/functions.metadata b/test/SdkE2ETests/Contents/functions.metadata index 7758da8f6..963dd07ea 100644 --- a/test/SdkE2ETests/Contents/functions.metadata +++ b/test/SdkE2ETests/Contents/functions.metadata @@ -22,6 +22,7 @@ "name": "myBlob", "type": "Blob", "direction": "In", + "dataType": "String", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage" }, @@ -59,6 +60,7 @@ "name": "myBlob", "type": "Blob", "direction": "In", + "dataType": "String", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage" } @@ -87,6 +89,7 @@ "name": "Name", "type": "Queue", "direction": "Out", + "dataType": "String", "queueName": "functionstesting2", "connection": "AzureWebJobsStorage" },