From 8d7494c6dfcd9a8ac2544d8efe30386f28ba27a5 Mon Sep 17 00:00:00 2001 From: pragnagopa Date: Tue, 19 May 2020 17:31:55 -0700 Subject: [PATCH 1/4] Add Custom Handler options --- .../Config/ConfigurationSectionNames.cs | 1 + .../Http/Configuration/HttpWorkerOptions.cs | 4 + .../Configuration/HttpWorkerOptionsSetup.cs | 97 +++++++-- .../Workers/Http/DefaultHttpWorkerService.cs | 19 +- .../HttpScriptInvocationResultExtensions.cs | 8 +- .../Workers/Http/HttpWorkerConstants.cs | 3 + .../Workers/Http/HttpWorkerContext.cs | 2 - .../Workers/Http/HttpWorkerDescription.cs | 36 +++- .../Workers/Http/HttpWorkerProcess.cs | 6 +- .../DefaultWorkerProcessFactory.cs | 13 +- .../ProcessManagement/WorkerDescription.cs | 38 +++- .../ProcessManagement/WorkerProcess.cs | 4 +- .../Configuration/RpcWorkerConfigFactory.cs | 20 +- .../Workers/Rpc/RpcWorkerDescription.cs | 11 -- .../SamplesEndToEndTests_HttpWorker.cs | 2 +- .../HttpWorkerOptionsSetupTests.cs | 187 +++++++++++++++--- .../DefaultHttpWorkerServiceTests.cs | 100 +++++++++- ...tpScriptInvocationResultExtensionsTests.cs | 32 +-- .../HttpWorker/HttpWorkerTestUtilities.cs | 26 ++- .../DefaultWorkerProcessFactoryTests.cs | 42 +++- .../Workers/Http/HttpWorkerProcessTests.cs | 10 +- .../Workers/Rpc/RpcWorkerConfigTests.cs | 5 - 22 files changed, 547 insertions(+), 119 deletions(-) diff --git a/src/WebJobs.Script/Config/ConfigurationSectionNames.cs b/src/WebJobs.Script/Config/ConfigurationSectionNames.cs index 4cf8933c9f..7ba20a8fae 100644 --- a/src/WebJobs.Script/Config/ConfigurationSectionNames.cs +++ b/src/WebJobs.Script/Config/ConfigurationSectionNames.cs @@ -16,6 +16,7 @@ public static class ConfigurationSectionNames public const string ManagedDependency = "managedDependency"; public const string Extensions = "extensions"; public const string HttpWorker = "httpWorker"; + public const string CustomHandler = "customHandler"; public const string Http = Extensions + ":http"; public const string Hsts = Http + ":hsts"; public const string CustomHttpHeaders = Http + ":customHeaders"; diff --git a/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptions.cs b/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptions.cs index 906e4767f5..d345b7d25a 100644 --- a/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptions.cs +++ b/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptions.cs @@ -5,10 +5,14 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Http { public class HttpWorkerOptions { + public string Type { get; set; } = "http"; + public HttpWorkerDescription Description { get; set; } public WorkerProcessArguments Arguments { get; set; } public int Port { get; set; } + + public bool EnableForwardingHttpRequest { get; set; } } } diff --git a/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs index 4292ad1c6d..c27939efc0 100644 --- a/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Collections.Generic; +using System.IO; using System.Net; using System.Net.Sockets; using Microsoft.Azure.WebJobs.Script.Configuration; @@ -8,6 +10,8 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace Microsoft.Azure.WebJobs.Script.Workers.Http { @@ -17,6 +21,8 @@ internal class HttpWorkerOptionsSetup : IConfigureOptions private ILogger _logger; private IMetricsLogger _metricsLogger; private ScriptJobHostOptions _scriptJobHostOptions; + private string argumentsSectionName = $"{WorkerConstants.WorkerDescription}:arguments"; + private string workerArgumentsSectionName = $"{WorkerConstants.WorkerDescription}:workerArguments"; public HttpWorkerOptionsSetup(IOptions scriptJobHostOptions, IConfiguration configuration, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger) { @@ -30,27 +36,92 @@ public void Configure(HttpWorkerOptions options) { IConfigurationSection jobHostSection = _configuration.GetSection(ConfigurationSectionNames.JobHost); var httpWorkerSection = jobHostSection.GetSection(ConfigurationSectionNames.HttpWorker); + var customHandlerSection = jobHostSection.GetSection(ConfigurationSectionNames.CustomHandler); + + if (httpWorkerSection.Exists() && customHandlerSection.Exists()) + { + _logger.LogWarning($"Both {ConfigurationSectionNames.HttpWorker} and {ConfigurationSectionNames.CustomHandler} sections are spefified in {ScriptConstants.HostMetadataFileName} file. {ConfigurationSectionNames.CustomHandler} takes precedence."); + } + + if (customHandlerSection.Exists()) + { + ConfigureWorkerDescription(options, customHandlerSection); + return; + } + if (httpWorkerSection.Exists()) { + // TODO: Add aka.ms/link to new docs _metricsLogger.LogEvent(MetricEventNames.HttpWorker); - httpWorkerSection.Bind(options); - HttpWorkerDescription httpWorkerDescription = options.Description; + _logger.LogWarning($"Section {ConfigurationSectionNames.HttpWorker} will be deprecated. Please use {ConfigurationSectionNames.CustomHandler} section."); + ConfigureWorkerDescription(options, httpWorkerSection); + // Explicity set this empty to differentiate between customHandler and httpWorker options. + options.Type = string.Empty; + } + } + + private void ConfigureWorkerDescription(HttpWorkerOptions options, IConfigurationSection workerSection) + { + workerSection.Bind(options); + HttpWorkerDescription httpWorkerDescription = options.Description; + + if (httpWorkerDescription == null) + { + throw new HostConfigurationException($"Missing worker Description."); + } + + var argumentsList = GetArgumentList(workerSection, argumentsSectionName); + if (argumentsList != null) + { + httpWorkerDescription.Arguments = argumentsList; + } + + var workerArgumentList = GetArgumentList(workerSection, workerArgumentsSectionName); + if (workerArgumentList != null) + { + httpWorkerDescription.WorkerArguments = workerArgumentList; + } + + httpWorkerDescription.ApplyDefaultsAndValidate(_scriptJobHostOptions.RootScriptPath, _logger); - if (httpWorkerDescription == null) + // Set default working directory to function app root. + if (string.IsNullOrEmpty(httpWorkerDescription.WorkingDirectory)) + { + httpWorkerDescription.WorkingDirectory = _scriptJobHostOptions.RootScriptPath; + } + else + { + // Compute working directory relative to fucntion app root. + if (!Path.IsPathRooted(httpWorkerDescription.WorkingDirectory)) { - throw new HostConfigurationException($"Missing WorkerDescription for HttpWorker"); + httpWorkerDescription.WorkingDirectory = Path.Combine(_scriptJobHostOptions.RootScriptPath, httpWorkerDescription.WorkingDirectory); } - httpWorkerDescription.ApplyDefaultsAndValidate(_scriptJobHostOptions.RootScriptPath, _logger); - options.Arguments = new WorkerProcessArguments() - { - ExecutablePath = options.Description.DefaultExecutablePath, - WorkerPath = options.Description.DefaultWorkerPath - }; + } + + options.Arguments = new WorkerProcessArguments() + { + ExecutablePath = options.Description.DefaultExecutablePath, + WorkerPath = options.Description.DefaultWorkerPath + }; + + options.Arguments.ExecutableArguments.AddRange(options.Description.Arguments); + options.Port = GetUnusedTcpPort(); + } - options.Arguments.ExecutableArguments.AddRange(options.Description.Arguments); - options.Port = GetUnusedTcpPort(); - _logger.LogDebug("Configured httpWorker with {DefaultExecutablePath}: {exepath} with arguments {args}", nameof(options.Description.DefaultExecutablePath), options.Description.DefaultExecutablePath, options.Arguments); + private static List GetArgumentList(IConfigurationSection httpWorkerSection, string argumentSectionName) + { + var argumentsSection = httpWorkerSection.GetSection(argumentSectionName); + if (argumentsSection.Exists() && argumentsSection?.Value != null) + { + try + { + return JsonConvert.DeserializeObject>(argumentsSection.Value); + } + catch + { + } } + return null; } internal static int GetUnusedTcpPort() diff --git a/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs b/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs index 83d93e71c7..63a2334028 100644 --- a/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs +++ b/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs @@ -40,7 +40,12 @@ public Task InvokeAsync(ScriptInvocationContext scriptInvocationContext) { if (scriptInvocationContext.FunctionMetadata.IsHttpInAndOutFunction()) { - return ProcessHttpInAndOutInvocationRequest(scriptInvocationContext); + // type is empty for httpWorker. Opt-in for custom handler section. + if (string.IsNullOrEmpty(_httpWorkerOptions.Type) || _httpWorkerOptions.EnableForwardingHttpRequest) + { + return ProcessHttpInAndOutInvocationRequest(scriptInvocationContext); + } + return ProcessDefaultInvocationRequest(scriptInvocationContext); } return ProcessDefaultInvocationRequest(scriptInvocationContext); } @@ -108,11 +113,11 @@ internal async Task ProcessDefaultInvocationRequest(ScriptInvocationContext scri { if (httpScriptInvocationResult.Outputs == null || !httpScriptInvocationResult.Outputs.Any()) { - _logger.LogDebug("Outputs not set on http response for invocationId:{invocationId}", scriptInvocationContext.ExecutionContext.InvocationId); + _logger.LogWarning("Outputs not set on http response for invocationId:{invocationId}", scriptInvocationContext.ExecutionContext.InvocationId); } if (httpScriptInvocationResult.ReturnValue == null) { - _logger.LogDebug("ReturnValue not set on http response for invocationId:{invocationId}", scriptInvocationContext.ExecutionContext.InvocationId); + _logger.LogWarning("ReturnValue not set on http response for invocationId:{invocationId}", scriptInvocationContext.ExecutionContext.InvocationId); } ProcessLogsFromHttpResponse(scriptInvocationContext, httpScriptInvocationResult); @@ -165,7 +170,13 @@ private HttpRequestMessage CreateAndGetHttpRequestMessage(string functionName, s private void AddRequestHeadersAndSetRequestUri(HttpRequestMessage httpRequestMessage, string functionName, string invocationId) { - httpRequestMessage.RequestUri = new Uri(new UriBuilder(WorkerConstants.HttpScheme, WorkerConstants.HostName, _httpWorkerOptions.Port, functionName).ToString()); + string pathValue = functionName; + // _httpWorkerOptions.Type is populated only in customHandler section + if (httpRequestMessage.RequestUri != null && !string.IsNullOrEmpty(_httpWorkerOptions.Type)) + { + pathValue = httpRequestMessage.RequestUri.AbsolutePath; + } + httpRequestMessage.RequestUri = new Uri(new UriBuilder(WorkerConstants.HttpScheme, WorkerConstants.HostName, _httpWorkerOptions.Port, pathValue).ToString()); httpRequestMessage.Headers.Add(HttpWorkerConstants.InvocationIdHeaderName, invocationId); httpRequestMessage.Headers.Add(HttpWorkerConstants.HostVersionHeaderName, ScriptHost.Version); httpRequestMessage.Headers.UserAgent.ParseAdd($"{HttpWorkerConstants.UserAgentHeaderValue}/{ScriptHost.Version}"); diff --git a/src/WebJobs.Script/Workers/Http/HttpScriptInvocationResultExtensions.cs b/src/WebJobs.Script/Workers/Http/HttpScriptInvocationResultExtensions.cs index 11403f75a6..736d3a60f7 100644 --- a/src/WebJobs.Script/Workers/Http/HttpScriptInvocationResultExtensions.cs +++ b/src/WebJobs.Script/Workers/Http/HttpScriptInvocationResultExtensions.cs @@ -3,12 +3,9 @@ using System; using System.Collections.Generic; -using System.Dynamic; using System.Linq; -using Microsoft.Azure.WebJobs.Script.Binding; using Microsoft.Azure.WebJobs.Script.Description; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; namespace Microsoft.Azure.WebJobs.Script.Workers.Http { @@ -33,7 +30,10 @@ public static ScriptInvocationResult ToScriptInvocationResult(this HttpScriptInv if (httpScriptInvocationResult.ReturnValue != null) { BindingMetadata returnParameterBindingMetadata = GetBindingMetadata(ScriptConstants.SystemReturnParameterBindingName, scriptInvocationContext); - scriptInvocationResult.Return = GetBindingValue(returnParameterBindingMetadata.DataType, httpScriptInvocationResult.ReturnValue); + if (returnParameterBindingMetadata != null) + { + scriptInvocationResult.Return = GetBindingValue(returnParameterBindingMetadata.DataType, httpScriptInvocationResult.ReturnValue); + } } return scriptInvocationResult; } diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerConstants.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerConstants.cs index 2cf3d33a5c..3925dc32b1 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerConstants.cs @@ -13,5 +13,8 @@ public static class HttpWorkerConstants // Child Process Env vars public const string PortEnvVarName = "FUNCTIONS_HTTPWORKER_PORT"; public const string WorkerIdEnvVarName = "FUNCTIONS_HTTPWORKER_ID"; + public const string FunctionAppRootVarName = "FUNCTIONS_APP_ROOT_PATH"; + public const string CustomHandlerPortEnvVarName = "FUNCTIONS_CUSTOMHANDLER_PORT"; + public const string CustomHandlerWorkerIdEnvVarName = "FUNCTIONS_CUSTOMHANDLER_WORKER_ID"; } } diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerContext.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerContext.cs index d1ce3d5086..413e7abb8c 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerContext.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerContext.cs @@ -1,8 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Microsoft.Azure.WebJobs.Script.Workers.Rpc; - namespace Microsoft.Azure.WebJobs.Script.Workers.Http { // Arguments to start a worker process diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs index ae89ea9e44..6a1eb58e42 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs @@ -5,26 +5,48 @@ using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.IO; +using System.Linq; using Microsoft.Extensions.Logging; namespace Microsoft.Azure.WebJobs.Script.Workers.Http { public class HttpWorkerDescription : WorkerDescription { - public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger) + /// + /// Gets or sets WorkingDirectory for the process: DefaultExecutablePath + /// + public string WorkingDirectory { get; set; } + + public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger) { - if (workerDirectory == null) + if (inputWorkerDirectory == null) { - throw new ArgumentNullException(nameof(workerDirectory)); + throw new ArgumentNullException(nameof(inputWorkerDirectory)); } Arguments = Arguments ?? new List(); + WorkerArguments = WorkerArguments ?? new List(); + + if (string.IsNullOrEmpty(WorkerDirectory)) + { + WorkerDirectory = inputWorkerDirectory; + } + else + { + if (!Path.IsPathRooted(WorkerDirectory)) + { + WorkerDirectory = Path.Combine(inputWorkerDirectory, WorkerDirectory); + } + } - WorkerDirectory = WorkerDirectory ?? workerDirectory; + ExpandEnvironmentVariables(); - // If DefaultWorkerPath is not set then compute full path for DefaultExecutablePath from scriptRootDir - if (string.IsNullOrEmpty(DefaultWorkerPath) && !string.IsNullOrEmpty(DefaultExecutablePath) && !Path.IsPathRooted(DefaultExecutablePath)) + // If DefaultWorkerPath is not set then compute full path for DefaultExecutablePath from WorkingDirectory. + // Empty DefaultWorkerPath or empty Arguments indicates DefaultExecutablePath is either a runtime on the system path or a file relative to WorkingDirectory. + // No need to find full path for DefaultWorkerPath as WorkerDirectory will be set when launching the worker process. + // DefaultWorkerPath can be specified as part of the arguments list + if (string.IsNullOrEmpty(DefaultWorkerPath) && !string.IsNullOrEmpty(DefaultExecutablePath) && !Path.IsPathRooted(DefaultExecutablePath) && !Arguments.Any()) { - DefaultExecutablePath = Path.Combine(workerDirectory, DefaultExecutablePath); + DefaultExecutablePath = Path.Combine(WorkerDirectory, DefaultExecutablePath); } // If DefaultWorkerPath is set and find full path from scriptRootDir diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerProcess.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerProcess.cs index 79d612dc0d..a4c3f07175 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerProcess.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerProcess.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.Diagnostics; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Eventing; @@ -50,11 +51,14 @@ internal override Process CreateWorkerProcess() RequestId = Guid.NewGuid().ToString(), WorkerId = _workerId, Arguments = _workerProcessArguments, - WorkingDirectory = _scriptRootPath, + WorkingDirectory = _httpWorkerOptions.Description.WorkingDirectory, Port = _httpWorkerOptions.Port }; workerContext.EnvironmentVariables.Add(HttpWorkerConstants.PortEnvVarName, _httpWorkerOptions.Port.ToString()); workerContext.EnvironmentVariables.Add(HttpWorkerConstants.WorkerIdEnvVarName, _workerId); + workerContext.EnvironmentVariables.Add(HttpWorkerConstants.CustomHandlerPortEnvVarName, _httpWorkerOptions.Port.ToString()); + workerContext.EnvironmentVariables.Add(HttpWorkerConstants.CustomHandlerWorkerIdEnvVarName, _workerId); + workerContext.EnvironmentVariables.Add(HttpWorkerConstants.FunctionAppRootVarName, _scriptRootPath); Process workerProcess = _processFactory.CreateWorkerProcess(workerContext); if (_environment.IsLinuxConsumption()) { diff --git a/src/WebJobs.Script/Workers/ProcessManagement/DefaultWorkerProcessFactory.cs b/src/WebJobs.Script/Workers/ProcessManagement/DefaultWorkerProcessFactory.cs index f2e7e9fe0a..0c790da4d1 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/DefaultWorkerProcessFactory.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/DefaultWorkerProcessFactory.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Text; @@ -34,19 +35,23 @@ public virtual Process CreateWorkerProcess(WorkerContext context) WorkingDirectory = context.WorkingDirectory, Arguments = GetArguments(context), }; - var processEnvVariables = context.EnvironmentVariables; if (processEnvVariables != null && processEnvVariables.Any()) { - foreach (var evnVar in processEnvVariables) + foreach (var envVar in processEnvVariables) { - startInfo.EnvironmentVariables[evnVar.Key] = evnVar.Value; + startInfo.EnvironmentVariables[envVar.Key] = envVar.Value; + startInfo.Arguments = startInfo.Arguments.Replace($"%{envVar.Key}%", envVar.Value); } } return new Process { StartInfo = startInfo }; } - private StringBuilder MergeArguments(StringBuilder builder, string arg) => builder.AppendFormat(" {0}", arg); + private StringBuilder MergeArguments(StringBuilder builder, string arg) + { + string expandedArg = Environment.ExpandEnvironmentVariables(arg); + return builder.AppendFormat(" {0}", expandedArg); + } public string GetArguments(WorkerContext context) { diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs index 48ec42ddec..7f66a5e8b0 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs @@ -1,13 +1,18 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using System.Collections.Generic; +using System.IO; using Microsoft.Extensions.Logging; namespace Microsoft.Azure.WebJobs.Script.Workers { public abstract class WorkerDescription { + // Can be replaced for testing purposes + internal Func FileExists { get; set; } = File.Exists; + /// /// Gets or sets the default executable path. /// @@ -24,10 +29,39 @@ public abstract class WorkerDescription public string WorkerDirectory { get; set; } /// - /// Gets or sets the command line args to pass to the worker + /// Gets or sets the command line args to pass to the worker. Will be appended after DefaultExecutablePath but before DefaultWorkerPath + /// + public IList Arguments { get; set; } + + /// + /// Gets or sets the command line args to pass to the worker. Will be appended after DefaultWorkerPath /// - public List Arguments { get; set; } + public IList WorkerArguments { get; set; } public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger); + + internal void ThrowIfFileNotExists(string inputFile, string paramName) + { + if (inputFile == null) + { + return; + } + if (!FileExists(inputFile)) + { + throw new FileNotFoundException($"File {paramName}: {inputFile} does not exist."); + } + } + + internal void ExpandEnvironmentVariables() + { + if (DefaultWorkerPath != null) + { + DefaultWorkerPath = Environment.ExpandEnvironmentVariables(DefaultWorkerPath); + } + if (DefaultExecutablePath != null) + { + DefaultExecutablePath = Environment.ExpandEnvironmentVariables(DefaultExecutablePath); + } + } } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs index f74836a8c5..a8dc18ba1f 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs @@ -54,7 +54,7 @@ public Task StartProcessAsync() _process.Exited += (sender, e) => OnProcessExited(sender, e); _process.EnableRaisingEvents = true; - _workerProcessLogger?.LogInformation($"Starting worker process:{_process.StartInfo.FileName} {_process.StartInfo.Arguments}"); + _workerProcessLogger?.LogInformation($"Starting worker process with FileName:{_process.StartInfo.FileName} WorkingDirectory:{_process.StartInfo.WorkingDirectory} Arguments:{_process.StartInfo.Arguments}"); _process.Start(); _workerProcessLogger?.LogInformation($"{_process.StartInfo.FileName} process with Id={_process.Id} started"); @@ -71,7 +71,7 @@ public Task StartProcessAsync() } catch (Exception ex) { - _workerProcessLogger.LogError(ex, "Failed to start Worker Channel"); + _workerProcessLogger.LogError(ex, $"Failed to start Worker Channel. Process fileName: {_process.StartInfo.FileName}"); return Task.FromException(ex); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index ef41169620..d20e7c0528 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -154,7 +154,7 @@ internal void AddProvider(string workerDir) if (ShouldAddWorkerConfig(workerDescription.Language)) { workerDescription.FormatWorkerPathIfNeeded(_systemRuntimeInformation, _environment, _logger); - workerDescription.ThrowIfDefaultWorkerPathNotExists(); + workerDescription.ThrowIfFileNotExists(workerDescription.DefaultWorkerPath, nameof(workerDescription.DefaultWorkerPath)); _workerDescripionDictionary[workerDescription.Language] = workerDescription; _logger.LogDebug($"Added WorkerConfig for language: {workerDescription.Language}"); } @@ -183,22 +183,6 @@ private static Dictionary GetWorkerDescriptionProfile return descriptionProfiles; } - private static WorkerDescription GetWorkerDescriptionFromProfiles(string key, Dictionary descriptionProfiles, RpcWorkerDescription defaultWorkerDescription) - { - RpcWorkerDescription profileDescription = null; - if (descriptionProfiles.TryGetValue(key, out profileDescription)) - { - profileDescription.Arguments = profileDescription.Arguments?.Count > 0 ? profileDescription.Arguments : defaultWorkerDescription.Arguments; - profileDescription.DefaultExecutablePath = string.IsNullOrEmpty(profileDescription.DefaultExecutablePath) ? defaultWorkerDescription.DefaultExecutablePath : profileDescription.DefaultExecutablePath; - profileDescription.DefaultWorkerPath = string.IsNullOrEmpty(profileDescription.DefaultWorkerPath) ? defaultWorkerDescription.DefaultWorkerPath : profileDescription.DefaultWorkerPath; - profileDescription.Extensions = profileDescription.Extensions ?? defaultWorkerDescription.Extensions; - profileDescription.Language = string.IsNullOrEmpty(profileDescription.Language) ? defaultWorkerDescription.Language : profileDescription.Language; - profileDescription.WorkerDirectory = string.IsNullOrEmpty(profileDescription.WorkerDirectory) ? defaultWorkerDescription.WorkerDirectory : profileDescription.WorkerDirectory; - return profileDescription; - } - return defaultWorkerDescription; - } - private static void GetWorkerDescriptionFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) { var defaultExecutablePathSetting = languageSection.GetSection($"{WorkerConstants.WorkerDescriptionDefaultExecutablePath}"); @@ -223,7 +207,7 @@ internal static void AddArgumentsFromAppSettings(RpcWorkerDescription workerDesc var argumentsSection = languageSection.GetSection($"{WorkerConstants.WorkerDescriptionArguments}"); if (argumentsSection.Value != null) { - workerDescription.Arguments.AddRange(Regex.Split(argumentsSection.Value, @"\s+")); + ((List)workerDescription.Arguments).AddRange(Regex.Split(argumentsSection.Value, @"\s+")); } } diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 59858fc4ba..ee7ef04ac2 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -73,9 +73,6 @@ public List Extensions } } - // Can be replaced for testing purposes - internal Func FileExists { private get; set; } = File.Exists; - public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger) { if (workerDirectory == null) @@ -122,14 +119,6 @@ internal void ValidateDefaultWorkerPathFormatters(ISystemRuntimeInformation syst } } - internal void ThrowIfDefaultWorkerPathNotExists() - { - if (!string.IsNullOrEmpty(DefaultWorkerPath) && !FileExists(DefaultWorkerPath)) - { - throw new FileNotFoundException($"Did not find {nameof(DefaultWorkerPath)} for language: {Language}"); - } - } - private void ValidateOSPlatform(OSPlatform os) { if (!SupportedOperatingSystems.Any(s => s.Equals(os.ToString(), StringComparison.OrdinalIgnoreCase))) diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_HttpWorker.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_HttpWorker.cs index d212649371..0719f17d82 100644 --- a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_HttpWorker.cs +++ b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_HttpWorker.cs @@ -29,7 +29,7 @@ public SamplesEndToEndTests_HttpWorker(TestFixture fixture) } [Fact] - public async Task HttpTrigger_PowerShell_Get_Succeeds() + public async Task HttpTrigger_HttpWorker_Get_Succeeds() { await InvokeHttpTrigger("HttpTrigger"); } diff --git a/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs index 371aa35eae..e8fbf2b223 100644 --- a/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs @@ -15,7 +15,6 @@ using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; using Xunit; -using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames; namespace Microsoft.Azure.WebJobs.Script.Tests.Configuration { @@ -44,9 +43,7 @@ public HttpWorkerOptionsSetupTests() FunctionTimeout = TimeSpan.FromSeconds(3) }; - _rootPath = Path.Combine(Environment.CurrentDirectory, "ScriptHostTests"); - Environment.SetEnvironmentVariable(AzureWebJobsScriptRoot, _rootPath); - + _rootPath = Path.Combine(Environment.CurrentDirectory, "HttpWorkerOptionsSetupTests"); if (!Directory.Exists(_rootPath)) { Directory.CreateDirectory(_rootPath); @@ -76,14 +73,23 @@ public HttpWorkerOptionsSetupTests() } } }")] + [InlineData(@"{ + 'version': '2.0', + 'customHandler': { + 'description': { + 'defaultExecutablePath': 'testExe' + } + } + }")] public void MissingOrValid_HttpWorkerConfig_DoesNotThrowException(string hostJsonContent) { File.WriteAllText(_hostJsonFile, hostJsonContent); var configuration = BuildHostJsonConfiguration(); HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); - var ex = Record.Exception(() => setup.Configure(options)); - Assert.Null(ex); + + setup.Configure(options); + if (options.Description != null && !string.IsNullOrEmpty(options.Description.DefaultExecutablePath)) { string expectedDefaultExecutablePath = Path.Combine(_scriptJobHostOptions.RootScriptPath, "testExe"); @@ -91,42 +97,70 @@ public void MissingOrValid_HttpWorkerConfig_DoesNotThrowException(string hostJso } } - [Fact] - public void InValid_HttpWorkerConfig_Throws_HostConfigurationException() - { - string hostJsonContent = @"{ + [Theory] + [InlineData(@"{ 'version': '2.0', 'httpWorker': { 'invalid': { 'defaultExecutablePath': 'testExe' } } - }"; + }")] + [InlineData(@"{ + 'version': '2.0', + 'httpWorker': { + 'description': { + 'langauge': 'testExe' + } + } + }")] + public void InValid_HttpWorkerConfig_Throws_Exception(string hostJsonContent) + { File.WriteAllText(_hostJsonFile, hostJsonContent); var configuration = BuildHostJsonConfiguration(); HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); - var ex = Assert.Throws(() => setup.Configure(options)); - Assert.Contains("Missing WorkerDescription for HttpWorker", ex.Message); + var ex = Record.Exception(() => setup.Configure(options)); + Assert.NotNull(ex); + if (options.Description == null) + { + Assert.IsType(ex); + Assert.Equal($"Missing worker Description.", ex.Message); + } + else + { + Assert.IsType(ex); + Assert.Equal($"WorkerDescription DefaultExecutablePath cannot be empty", ex.Message); + } } [Fact] - public void InValid_HttpWorkerConfig_Throws_ValidationException() + public void CustomHandlerConfig_ExpandEnvVars() { string hostJsonContent = @"{ 'version': '2.0', - 'httpWorker': { + 'customHandler': { 'description': { - 'langauge': 'testExe' + 'defaultExecutablePath': '%TestEnv%', + 'defaultWorkerPath': '%TestEnv%' } } }"; - File.WriteAllText(_hostJsonFile, hostJsonContent); - var configuration = BuildHostJsonConfiguration(); + try + { + Environment.SetEnvironmentVariable("TestEnv", "TestVal"); + File.WriteAllText(_hostJsonFile, hostJsonContent); + var configuration = BuildHostJsonConfiguration(); HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); - HttpWorkerOptions options = new HttpWorkerOptions(); - var ex = Assert.Throws(() => setup.Configure(options)); - Assert.Contains("WorkerDescription DefaultExecutablePath cannot be empty", ex.Message); + HttpWorkerOptions options = new HttpWorkerOptions(); + setup.Configure(options); + Assert.Equal("TestVal", options.Description.DefaultExecutablePath); + Assert.Contains("TestVal", options.Description.DefaultWorkerPath); + } + finally + { + Environment.SetEnvironmentVariable("TestEnv", string.Empty); + } } [Theory] @@ -148,7 +182,7 @@ public void InValid_HttpWorkerConfig_Throws_ValidationException() 'defaultExecutablePath': 'node' } } - }", true, false, false)] + }", false, false, false)] [InlineData(@"{ 'version': '2.0', 'httpWorker': { @@ -174,9 +208,9 @@ public void HttpWorker_Config_ExpectedValues(string hostJsonContent, bool append var configuration = BuildHostJsonConfiguration(); HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); - setup.Configure(options); Assert.True(_metricsLogger.LoggedEvents.Contains(MetricEventNames.HttpWorker)); + setup.Configure(options); //Verify worker exe path is expected if (appendCurrentDirectoryToExe) { @@ -209,6 +243,109 @@ public void HttpWorker_Config_ExpectedValues(string hostJsonContent, bool append Assert.Equal("--xTest1 --xTest2", options.Description.Arguments[0]); } + [Theory] + [InlineData(@"{ + 'version': '2.0', + 'customHandler': { + 'description': { + 'defaultExecutablePath': 'node', + 'arguments': ['httpWorker.js'], + 'workingDirectory': 'c:/myWorkingDir', + 'workerDirectory': 'c:/myWorkerDir' + } + } + }", false, false, false)] + [InlineData(@"{ + 'version': '2.0', + 'customHandler': { + 'description': { + 'defaultExecutablePath': 'node', + 'workingDirectory': 'myWorkingDir', + 'workerDirectory': 'myWorkerDir' + } + } + }", true, true, true)] + public void CustomHandler_Config_ExpectedValues_WorkerDirectory_WorkingDirectory(string hostJsonContent, bool appendCurrentDirToDefaultExe, bool appendCurrentDirToWorkingDir, bool appendCurrentDirToWorkerDir) + { + File.WriteAllText(_hostJsonFile, hostJsonContent); + var configuration = BuildHostJsonConfiguration(); + HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory); + HttpWorkerOptions options = new HttpWorkerOptions(); + setup.Configure(options); + //Verify worker exe path is expected + if (appendCurrentDirToDefaultExe) + { + Assert.Equal(Path.Combine(_scriptJobHostOptions.RootScriptPath, "myWorkerDir", "node"), options.Description.DefaultExecutablePath); + } + else + { + Assert.Equal("node", options.Description.DefaultExecutablePath); + } + + // Verify worker dir is expected + if (appendCurrentDirToWorkerDir) + { + Assert.Equal(Path.Combine(_scriptJobHostOptions.RootScriptPath, "myWorkerDir"), options.Description.WorkerDirectory); + } + else + { + Assert.Equal(@"c:/myWorkerDir", options.Description.WorkerDirectory); + } + + //Verify workering Dir is expected + if (appendCurrentDirToWorkingDir) + { + Assert.Equal(Path.Combine(_scriptJobHostOptions.RootScriptPath, "myWorkingDir"), options.Description.WorkingDirectory); + } + else + { + Assert.Equal(@"c:/myWorkingDir", options.Description.WorkingDirectory); + } + } + + [Fact] + public void HttpWorkerConfig_OverrideConfigViaEnvVars_Test() + { + string hostJsonContent = @"{ + 'version': '2.0', + 'httpWorker': { + 'description': { + 'langauge': 'testExe', + 'defaultExecutablePath': 'dotnet', + 'defaultWorkerPath':'ManualTrigger/run.csx', + 'arguments': ['--xTest1 --xTest2'], + 'workerArguments': ['--xTest3 --xTest4'] + } + } + }"; + try + { + File.WriteAllText(_hostJsonFile, hostJsonContent); + Environment.SetEnvironmentVariable("AzureFunctionsJobHost:httpWorker:description:defaultWorkerPath", "OneSecondTimer/run.csx"); + Environment.SetEnvironmentVariable("AzureFunctionsJobHost:httpWorker:description:arguments", "[\"--xTest5\", \"--xTest6\", \"--xTest7\"]"); + var configuration = BuildHostJsonConfiguration(); + HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory); + HttpWorkerOptions options = new HttpWorkerOptions(); + setup.Configure(options); + Assert.Equal("dotnet", options.Description.DefaultExecutablePath); + // Verify options are overridden + Assert.Contains("OneSecondTimer/run.csx", options.Description.DefaultWorkerPath); + Assert.Equal(3, options.Description.Arguments.Count); + Assert.Contains("--xTest5", options.Description.Arguments); + Assert.Contains("--xTest6", options.Description.Arguments); + Assert.Contains("--xTest7", options.Description.Arguments); + + // Verify options not overridden + Assert.Equal(1, options.Description.WorkerArguments.Count); + Assert.Equal("--xTest3 --xTest4", options.Description.WorkerArguments.ElementAt(0)); + } + finally + { + Environment.SetEnvironmentVariable("AzureFunctionsJobHost:httpWorker:description:defaultWorkerPath", string.Empty); + Environment.SetEnvironmentVariable("AzureFunctionsJobHost:httpWorker:description:arguments", string.Empty); + } + } + [Fact] public void GetUnusedTcpPort_Succeeds() { @@ -228,14 +365,14 @@ public void GetUnusedTcpPort_Succeeds() private IConfiguration BuildHostJsonConfiguration(IEnvironment environment = null) { environment = environment ?? new TestEnvironment(); - var loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(_loggerProvider); var configSource = new HostJsonFileConfigurationSource(_options, environment, loggerFactory, new TestMetricsLogger()); var configurationBuilder = new ConfigurationBuilder() - .Add(configSource); + .Add(configSource) + .Add(new ScriptEnvironmentVariablesConfigurationSource()); return configurationBuilder.Build(); } diff --git a/test/WebJobs.Script.Tests/HttpWorker/DefaultHttpWorkerServiceTests.cs b/test/WebJobs.Script.Tests/HttpWorker/DefaultHttpWorkerServiceTests.cs index 1367db17de..df5e6c24f0 100644 --- a/test/WebJobs.Script.Tests/HttpWorker/DefaultHttpWorkerServiceTests.cs +++ b/test/WebJobs.Script.Tests/HttpWorker/DefaultHttpWorkerServiceTests.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Script.Extensions; using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Http; using Microsoft.Extensions.Logging; @@ -18,6 +19,7 @@ using Moq; using Moq.Protected; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Xunit; namespace Microsoft.Azure.WebJobs.Script.Tests.HttpWorker @@ -40,7 +42,8 @@ public DefaultHttpWorkerServiceTests() _testInvocationId = Guid.NewGuid(); _httpWorkerOptions = new HttpWorkerOptions() { - Port = _defaultPort + Port = _defaultPort, + Type = string.Empty }; } @@ -78,6 +81,38 @@ public async Task ProcessDefaultInvocationRequest_Succeeds() Assert.Equal(expectedHttpScriptInvocationResult.ReturnValue, invocationResult.Return); } + [Fact] + public async Task ProcessDefaultInvocationRequest_CustomHandler_EnableRequestForwarding_False() + { + var handlerMock = new Mock(MockBehavior.Strict); + var customHandlerOptions = new HttpWorkerOptions() + { + Port = _defaultPort, + Type = "http" + }; + handlerMock.Protected().Setup>("SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .Callback((request, token) => ValidateeSimpleHttpTriggerSentAsDefaultInvocationRequest(request)) + .ReturnsAsync(HttpWorkerTestUtilities.GetValidHttpResponseMessageWithJsonRes()); + + _httpClient = new HttpClient(handlerMock.Object); + _defaultHttpWorkerService = new DefaultHttpWorkerService(_httpClient, new OptionsWrapper(customHandlerOptions), _testLogger); + var testScriptInvocationContext = HttpWorkerTestUtilities.GetSimpleHttpTriggerScriptInvocationContext(TestFunctionName, _testInvocationId, _functionLogger); + await _defaultHttpWorkerService.InvokeAsync(testScriptInvocationContext); + var invocationResult = await testScriptInvocationContext.ResultSource.Task; + + var expectedHttpScriptInvocationResult = HttpWorkerTestUtilities.GetHttpScriptInvocationResultWithJsonRes(); + var testLogs = _functionLogger.GetLogMessages(); + Assert.True(testLogs.Count() == expectedHttpScriptInvocationResult.Logs.Count()); + Assert.True(testLogs.All(m => m.FormattedMessage.Contains("invocation log"))); + Assert.Equal(expectedHttpScriptInvocationResult.Outputs.Count(), invocationResult.Outputs.Count()); + Assert.Equal(expectedHttpScriptInvocationResult.ReturnValue, invocationResult.Return); + var responseJson = JObject.Parse(invocationResult.Outputs["res"].ToString()); + Assert.Equal("my world", responseJson["Body"]); + Assert.Equal("201", responseJson["StatusCode"]); + } + [Fact] public async Task ProcessDefaultInvocationRequest_DataType_Binary_Succeeds() { @@ -186,6 +221,45 @@ public async Task ProcessSimpleHttpTriggerInvocationRequest_Succeeds() Assert.Equal(expectedResponseContent, await response.Content.ReadAsStringAsync()); } + [Fact] + public async Task ProcessSimpleHttpTriggerInvocationRequest_CustomHandler_EnableForwardingHttpRequest_True() + { + var handlerMock = new Mock(MockBehavior.Strict); + var customHandlerOptions = new HttpWorkerOptions() + { + Port = _defaultPort, + Type = "http", + EnableForwardingHttpRequest = true + }; + handlerMock.Protected().Setup>("SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .Callback((request, token) => ValidateSimpleHttpTriggerInvocationRequest(request)) + .ReturnsAsync(HttpWorkerTestUtilities.GetValidSimpleHttpResponseMessage()); + + _httpClient = new HttpClient(handlerMock.Object); + _defaultHttpWorkerService = new DefaultHttpWorkerService(_httpClient, new OptionsWrapper(customHandlerOptions), _testLogger); + var testScriptInvocationContext = HttpWorkerTestUtilities.GetSimpleHttpTriggerScriptInvocationContext(TestFunctionName, _testInvocationId, _functionLogger); + await _defaultHttpWorkerService.InvokeAsync(testScriptInvocationContext); + var invocationResult = await testScriptInvocationContext.ResultSource.Task; + var expectedHttpResponseMessage = HttpWorkerTestUtilities.GetValidSimpleHttpResponseMessage(); + var expectedResponseContent = await expectedHttpResponseMessage.Content.ReadAsStringAsync(); + + var testLogs = _functionLogger.GetLogMessages(); + Assert.Equal(0, testLogs.Count()); + + Assert.Equal(1, invocationResult.Outputs.Count()); + var httpOutputResponse = invocationResult.Outputs.FirstOrDefault().Value as HttpResponseMessage; + Assert.NotNull(httpOutputResponse); + Assert.Equal(expectedHttpResponseMessage.StatusCode, httpOutputResponse.StatusCode); + Assert.Equal(expectedResponseContent, await httpOutputResponse.Content.ReadAsStringAsync()); + + var response = invocationResult.Return as HttpResponseMessage; + Assert.NotNull(response); + Assert.Equal(expectedHttpResponseMessage.StatusCode, response.StatusCode); + Assert.Equal(expectedResponseContent, await response.Content.ReadAsStringAsync()); + } + [Fact] public async Task ProcessSimpleHttpTriggerInvocationRequest_Sets_ExpectedResult() { @@ -369,6 +443,30 @@ private async void ValidateDefaultInvocationRequest(HttpRequestMessage httpReque } } + private async void ValidateeSimpleHttpTriggerSentAsDefaultInvocationRequest(HttpRequestMessage httpRequestMessage) + { + Assert.Contains($"{HttpWorkerConstants.UserAgentHeaderValue}/{ScriptHost.Version}", httpRequestMessage.Headers.UserAgent.ToString()); + Assert.Equal(_testInvocationId.ToString(), httpRequestMessage.Headers.GetValues(HttpWorkerConstants.InvocationIdHeaderName).Single()); + Assert.Equal(ScriptHost.Version, httpRequestMessage.Headers.GetValues(HttpWorkerConstants.HostVersionHeaderName).Single()); + Assert.Equal(httpRequestMessage.RequestUri.ToString(), $"http://127.0.0.1:{_defaultPort}/{TestFunctionName}"); + + HttpScriptInvocationContext httpScriptInvocationContext = await httpRequestMessage.Content.ReadAsAsync(); + + // Verify Metadata + var expectedMetadata = HttpWorkerTestUtilities.GetScriptInvocationBindingData(); + Assert.Equal(expectedMetadata.Count(), httpScriptInvocationContext.Metadata.Count()); + foreach (var key in expectedMetadata.Keys) + { + Assert.Equal(JsonConvert.SerializeObject(expectedMetadata[key]), httpScriptInvocationContext.Metadata[key]); + } + + // Verify Data + Assert.True(httpScriptInvocationContext.Data.Keys.Contains("testInputReq")); + JObject resultHttpReq = JObject.FromObject(httpScriptInvocationContext.Data["testInputReq"]); + JObject expectedHttpRequest = await HttpWorkerTestUtilities.GetTestHttpRequest().GetRequestAsJObject(); + Assert.True(JToken.DeepEquals(expectedHttpRequest, resultHttpReq)); + } + private async void ValidateSimpleHttpTriggerInvocationRequest(HttpRequestMessage httpRequestMessage) { Assert.Contains($"{HttpWorkerConstants.UserAgentHeaderValue}/{ScriptHost.Version}", httpRequestMessage.Headers.UserAgent.ToString()); diff --git a/test/WebJobs.Script.Tests/HttpWorker/HttpScriptInvocationResultExtensionsTests.cs b/test/WebJobs.Script.Tests/HttpWorker/HttpScriptInvocationResultExtensionsTests.cs index 43bc9bafe2..5321c0ef11 100644 --- a/test/WebJobs.Script.Tests/HttpWorker/HttpScriptInvocationResultExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/HttpWorker/HttpScriptInvocationResultExtensionsTests.cs @@ -3,21 +3,9 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Sockets; -using System.Text; -using System.Threading; using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Net.Http.Headers; -using Moq; -using Moq.Protected; -using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Xunit; namespace Microsoft.Azure.WebJobs.Script.Tests.HttpWorker @@ -36,5 +24,23 @@ public void GetHttpOutputBindingResponse_ReturnsExpected(string inputString, str var actualResult = HttpScriptInvocationResultExtensions.GetHttpOutputBindingResponse("httpOutput1", outputsFromWorker); Assert.Equal(expectedOutput, actualResult); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ToScripInvocationResultTests(bool includeReturnValue) + { + var testInvocationContext = HttpWorkerTestUtilities.GetSimpleHttpTriggerScriptInvocationContext("test", Guid.NewGuid(), new TestLogger("test")); + var testHttpInvocationResult = HttpWorkerTestUtilities.GetHttpScriptInvocationResultWithJsonRes(); + if (includeReturnValue) + { + testHttpInvocationResult.ReturnValue = "Hello return"; + } + var result = testHttpInvocationResult.ToScriptInvocationResult(testInvocationContext); + Assert.Null(result.Return); + + var resResult = JObject.Parse((string)result.Outputs["res"]); + Assert.Equal("my world", resResult["Body"]); + } } } diff --git a/test/WebJobs.Script.Tests/HttpWorker/HttpWorkerTestUtilities.cs b/test/WebJobs.Script.Tests/HttpWorker/HttpWorkerTestUtilities.cs index a770fa1bc6..12e1be7994 100644 --- a/test/WebJobs.Script.Tests/HttpWorker/HttpWorkerTestUtilities.cs +++ b/test/WebJobs.Script.Tests/HttpWorker/HttpWorkerTestUtilities.cs @@ -35,7 +35,6 @@ public static HttpRequest GetTestHttpRequest() httpRequest.Query = GetTestQueryParams(); httpRequest.Headers[HeaderNames.AcceptCharset] = UTF8AcceptCharset; httpRequest.Headers[HeaderNames.Accept] = AcceptHeaderValue; - var json = JsonConvert.SerializeObject(HttpContentStringValue); var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); httpRequest.Body = stream; @@ -51,7 +50,7 @@ public static QueryCollection GetTestQueryParams() public static List<(string name, DataType type, object val)> GetSimpleHttpTriggerScriptInvocationInputs() { List<(string name, DataType type, object val)> inputs = new List<(string name, DataType type, object val)>(); - inputs.Add(("myqueueItem", DataType.String, GetTestHttpRequest())); + inputs.Add(("testInputReq", DataType.String, GetTestHttpRequest())); return inputs; } @@ -130,6 +129,14 @@ public static HttpResponseMessage GetValidHttpResponseMessage() }; } + public static HttpResponseMessage GetValidHttpResponseMessageWithJsonRes() + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ObjectContent(GetHttpScriptInvocationResultWithJsonRes(), new JsonMediaTypeFormatter()) + }; + } + public static HttpResponseMessage GetValidHttpResponseMessage_DataType_Binary_Data() { return new HttpResponseMessage(HttpStatusCode.OK) @@ -242,6 +249,21 @@ public static ScriptInvocationContext GetScriptInvocationContext(string function return scriptInvocationContext; } + public static HttpScriptInvocationResult GetHttpScriptInvocationResultWithJsonRes() + { + JObject httpRes = new JObject(); + httpRes["statusCode"] = "201"; + httpRes["body"] = "my world"; + return new HttpScriptInvocationResult() + { + Logs = new List() { "invocation log1", "invocation log2" }, + Outputs = new Dictionary() + { + { "res", httpRes } + } + }; + } + public static ScriptInvocationContext GetSimpleHttpTriggerScriptInvocationContext(string functionName, Guid invocationId, ILogger testLogger) { ScriptInvocationContext scriptInvocationContext = new ScriptInvocationContext() diff --git a/test/WebJobs.Script.Tests/Workers/DefaultWorkerProcessFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/DefaultWorkerProcessFactoryTests.cs index cf5a3ec7bf..50c8b5dc5d 100644 --- a/test/WebJobs.Script.Tests/Workers/DefaultWorkerProcessFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/DefaultWorkerProcessFactoryTests.cs @@ -17,8 +17,36 @@ public static IEnumerable TestWorkerContexts { get { - yield return new object[] { new HttpWorkerContext() { Arguments = new WorkerProcessArguments() { ExecutablePath = "test" }, Port = 456, EnvironmentVariables = new Dictionary() { { "httpkey1", "httpvalue1" }, { "httpkey2", "httpvalue2" } } } }; - yield return new object[] { new RpcWorkerContext("testId", 500, "testWorkerId", new WorkerProcessArguments() { ExecutablePath = "test" }, "c:\testDir", new Uri("http://localhost")) }; + yield return new object[] + { + new HttpWorkerContext() + { + Arguments = new WorkerProcessArguments() + { + ExecutablePath = "test", + ExecutableArguments = new List() { "%httpkey1%", "%TestEnv%" }, + WorkerArguments = new List() { "%httpkey2%" } + }, + Port = 456, + EnvironmentVariables = new Dictionary() { { "httpkey1", "httpvalue1" }, { "httpkey2", "httpvalue2" } } + } + }; + yield return new object[] + { + new RpcWorkerContext( + "testId", + 500, + "testWorkerId", + new WorkerProcessArguments() + { + ExecutablePath = "test", + ExecutableArguments = new List() { "%httpkey1%", "%TestEnv%" }, + WorkerArguments = new List() { "%httpkey2%" } + }, + "c:\testDir", + new Uri("http://localhost"), + new Dictionary() { { "httpkey1", "httpvalue1" }, { "httpkey2", "httpvalue2" } }) + }; } } @@ -34,6 +62,7 @@ public static IEnumerable InvalidWorkerContexts [MemberData(nameof(TestWorkerContexts))] public void DefaultWorkerProcessFactory_Returns_ExpectedProcess(WorkerContext workerContext) { + Environment.SetEnvironmentVariable("TestEnv", "TestVal"); DefaultWorkerProcessFactory defaultWorkerProcessFactory = new DefaultWorkerProcessFactory(); Process childProcess = defaultWorkerProcessFactory.CreateWorkerProcess(workerContext); @@ -45,7 +74,16 @@ public void DefaultWorkerProcessFactory_Returns_ExpectedProcess(WorkerContext wo { Assert.Equal(expectedEnvVars[envVar.Key], actualEnvVars[envVar.Key]); } + if (workerContext is RpcWorkerContext) + { + Assert.Equal(" httpvalue1 TestVal httpvalue2 --host localhost --port 80 --workerId testWorkerId --requestId testId --grpcMaxMessageLength 2147483647", childProcess.StartInfo.Arguments); + } + else + { + Assert.Equal(" httpvalue1 TestVal httpvalue2", childProcess.StartInfo.Arguments); + } childProcess.Dispose(); + Environment.SetEnvironmentVariable("TestEnv", string.Empty); } [Theory] diff --git a/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerProcessTests.cs b/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerProcessTests.cs index f4650f8af7..850405fc4d 100644 --- a/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerProcessTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerProcessTests.cs @@ -31,7 +31,11 @@ public HttpWorkerProcessTests() _httpWorkerOptions = new HttpWorkerOptions() { Port = _workerPort, - Arguments = new WorkerProcessArguments() { ExecutablePath = "test" } + Arguments = new WorkerProcessArguments() { ExecutablePath = "test" }, + Description = new HttpWorkerDescription() + { + WorkingDirectory = @"c:\testDir" + } }; _settingsManager = ScriptSettingsManager.Instance; } @@ -53,6 +57,8 @@ public void CreateWorkerProcess_VerifyEnvVars(string processEnvValue) Assert.NotNull(childProcess.StartInfo.EnvironmentVariables); Assert.Equal(childProcess.StartInfo.EnvironmentVariables[HttpWorkerConstants.PortEnvVarName], _workerPort.ToString()); Assert.Equal(childProcess.StartInfo.EnvironmentVariables[HttpWorkerConstants.WorkerIdEnvVarName], _testWorkerId); + Assert.Equal(childProcess.StartInfo.EnvironmentVariables[HttpWorkerConstants.CustomHandlerPortEnvVarName], _workerPort.ToString()); + Assert.Equal(childProcess.StartInfo.EnvironmentVariables[HttpWorkerConstants.CustomHandlerWorkerIdEnvVarName], _testWorkerId); childProcess.Dispose(); } } @@ -69,4 +75,4 @@ public void CreateWorkerProcess_LinuxConsumption_AssingnsExecutePermissions_invo Assert.Contains("Error while assigning execute permission", testLogs[0].FormattedMessage); } } -} +} \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index b93f29a926..b3dc7741b2 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -113,11 +113,6 @@ public void ReadWorkerProviderFromConfig_ArgumentsFromSettings() public void ReadWorkerProviderFromConfig_EmptyWorkerPath() { var configs = new List() { MakeTestConfig(testLanguage, new string[0], false, string.Empty, true) }; - // Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig - Dictionary keyValuePairs = new Dictionary - { - [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{testLanguage}:{WorkerConstants.WorkerDescriptionArguments}"] = "--inspect=5689 --no-deprecation" - }; TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); var workerConfigs = TestReadWorkerProviderFromConfig(configs, new TestLogger(testLanguage), testMetricsLogger); From ab4b5d046a3fcb40d83f595ad443da7e0374a00d Mon Sep 17 00:00:00 2001 From: pragnagopa Date: Fri, 5 Jun 2020 13:42:12 -0700 Subject: [PATCH 2/4] Rebase and addressing CR --- .../Config/HostJsonFileConfigurationSource.cs | 2 +- src/WebJobs.Script/Diagnostics/MetricEventNames.cs | 2 +- .../Http/Configuration/HttpWorkerOptionsSetup.cs | 2 +- .../Workers/ProcessManagement/WorkerProcess.cs | 4 ++-- .../Configuration/HttpWorkerOptionsSetupTests.cs | 12 +++++++----- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs index fa75220907..5f4ff27eb7 100644 --- a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs +++ b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs @@ -51,7 +51,7 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider { "version", "functionTimeout", "functions", "http", "watchDirectories", "queues", "serviceBus", "eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies", - "httpWorker" + "customHandler" }; private readonly HostJsonFileConfigurationSource _configurationSource; diff --git a/src/WebJobs.Script/Diagnostics/MetricEventNames.cs b/src/WebJobs.Script/Diagnostics/MetricEventNames.cs index 4c3ec801d3..90984183fb 100644 --- a/src/WebJobs.Script/Diagnostics/MetricEventNames.cs +++ b/src/WebJobs.Script/Diagnostics/MetricEventNames.cs @@ -56,7 +56,7 @@ public static class MetricEventNames public const string FunctionInvokeFailed = "function.invoke.failed"; // Http worker events - public const string HttpWorker = "hostjsonfileconfigurationsource.httpworker"; + public const string CustomHandlerConfiguration = "hostjsonfileconfigurationsource.customhandler"; public const string DelayUntilWorkerIsInitialized = "httpworkerchannel.delayuntilworkerisinitialized"; // Out of proc process events diff --git a/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs index c27939efc0..b8475ed6c0 100644 --- a/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Http/Configuration/HttpWorkerOptionsSetup.cs @@ -45,6 +45,7 @@ public void Configure(HttpWorkerOptions options) if (customHandlerSection.Exists()) { + _metricsLogger.LogEvent(MetricEventNames.CustomHandlerConfiguration); ConfigureWorkerDescription(options, customHandlerSection); return; } @@ -52,7 +53,6 @@ public void Configure(HttpWorkerOptions options) if (httpWorkerSection.Exists()) { // TODO: Add aka.ms/link to new docs - _metricsLogger.LogEvent(MetricEventNames.HttpWorker); _logger.LogWarning($"Section {ConfigurationSectionNames.HttpWorker} will be deprecated. Please use {ConfigurationSectionNames.CustomHandler} section."); ConfigureWorkerDescription(options, httpWorkerSection); // Explicity set this empty to differentiate between customHandler and httpWorker options. diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs index a8dc18ba1f..6758cd1dda 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs @@ -54,7 +54,7 @@ public Task StartProcessAsync() _process.Exited += (sender, e) => OnProcessExited(sender, e); _process.EnableRaisingEvents = true; - _workerProcessLogger?.LogInformation($"Starting worker process with FileName:{_process.StartInfo.FileName} WorkingDirectory:{_process.StartInfo.WorkingDirectory} Arguments:{_process.StartInfo.Arguments}"); + _workerProcessLogger?.LogInformation($"Starting worker process with FileName:{_process.StartInfo.FileName} WorkingDirectory:{_process.StartInfo.WorkingDirectory} Arguments:{_process.StartInfo.Arguments}"); _process.Start(); _workerProcessLogger?.LogInformation($"{_process.StartInfo.FileName} process with Id={_process.Id} started"); @@ -71,7 +71,7 @@ public Task StartProcessAsync() } catch (Exception ex) { - _workerProcessLogger.LogError(ex, $"Failed to start Worker Channel. Process fileName: {_process.StartInfo.FileName}"); + _workerProcessLogger.LogError(ex, $"Failed to start Worker Channel. Process fileName: {_process.StartInfo.FileName}"); return Task.FromException(ex); } } diff --git a/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs index e8fbf2b223..6103b9d6a6 100644 --- a/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/HttpWorkerOptionsSetupTests.cs @@ -151,7 +151,7 @@ public void CustomHandlerConfig_ExpandEnvVars() Environment.SetEnvironmentVariable("TestEnv", "TestVal"); File.WriteAllText(_hostJsonFile, hostJsonContent); var configuration = BuildHostJsonConfiguration(); - HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); + HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); setup.Configure(options); Assert.Equal("TestVal", options.Description.DefaultExecutablePath); @@ -208,9 +208,8 @@ public void HttpWorker_Config_ExpectedValues(string hostJsonContent, bool append var configuration = BuildHostJsonConfiguration(); HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); - Assert.True(_metricsLogger.LoggedEvents.Contains(MetricEventNames.HttpWorker)); - setup.Configure(options); + //Verify worker exe path is expected if (appendCurrentDirectoryToExe) { @@ -269,9 +268,12 @@ public void CustomHandler_Config_ExpectedValues_WorkerDirectory_WorkingDirectory { File.WriteAllText(_hostJsonFile, hostJsonContent); var configuration = BuildHostJsonConfiguration(); - HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory); + HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); setup.Configure(options); + + Assert.True(_metricsLogger.LoggedEvents.Contains(MetricEventNames.CustomHandlerConfiguration)); + //Verify worker exe path is expected if (appendCurrentDirToDefaultExe) { @@ -324,7 +326,7 @@ public void HttpWorkerConfig_OverrideConfigViaEnvVars_Test() Environment.SetEnvironmentVariable("AzureFunctionsJobHost:httpWorker:description:defaultWorkerPath", "OneSecondTimer/run.csx"); Environment.SetEnvironmentVariable("AzureFunctionsJobHost:httpWorker:description:arguments", "[\"--xTest5\", \"--xTest6\", \"--xTest7\"]"); var configuration = BuildHostJsonConfiguration(); - HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory); + HttpWorkerOptionsSetup setup = new HttpWorkerOptionsSetup(new OptionsWrapper(_scriptJobHostOptions), configuration, _testLoggerFactory, _metricsLogger); HttpWorkerOptions options = new HttpWorkerOptions(); setup.Configure(options); Assert.Equal("dotnet", options.Description.DefaultExecutablePath); From aa3a5a30a658a7efcacadb89482411f05e8526ab Mon Sep 17 00:00:00 2001 From: pragnagopa Date: Fri, 5 Jun 2020 16:49:00 -0700 Subject: [PATCH 3/4] Add httpWorker to WellKnownProperties --- src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs index 5f4ff27eb7..ec12051b58 100644 --- a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs +++ b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs @@ -51,7 +51,7 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider { "version", "functionTimeout", "functions", "http", "watchDirectories", "queues", "serviceBus", "eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies", - "customHandler" + "customHandler", "httpWorker" }; private readonly HostJsonFileConfigurationSource _configurationSource; From 2eafcf7f2435d4772c4505b8bfc0b217e2ccae96 Mon Sep 17 00:00:00 2001 From: pragnagopa Date: Fri, 5 Jun 2020 16:51:33 -0700 Subject: [PATCH 4/4] fix comments --- src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs b/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs index 63a2334028..2feaf9f538 100644 --- a/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs +++ b/src/WebJobs.Script/Workers/Http/DefaultHttpWorkerService.cs @@ -40,7 +40,7 @@ public Task InvokeAsync(ScriptInvocationContext scriptInvocationContext) { if (scriptInvocationContext.FunctionMetadata.IsHttpInAndOutFunction()) { - // type is empty for httpWorker. Opt-in for custom handler section. + // type is empty for httpWorker section. EnableForwardingHttpRequest is opt-in for custom handler section. if (string.IsNullOrEmpty(_httpWorkerOptions.Type) || _httpWorkerOptions.EnableForwardingHttpRequest) { return ProcessHttpInAndOutInvocationRequest(scriptInvocationContext);