From d500b5097dfddaa53ce7ee47524dda0219efe6cd Mon Sep 17 00:00:00 2001 From: Brett Samblanet Date: Fri, 24 Aug 2018 10:21:55 -0700 Subject: [PATCH] throwing exception if no version property in host.json does not equal '2.0' --- sample/host.json | 1 + .../WebHostServiceCollectionExtensions.cs | 1 - .../WebJobsApplicationBuilderExtension.cs | 6 +- .../WebJobsScriptHostService.cs | 52 ++++--- .../Config/HostJsonFileConfigurationSource.cs | 61 ++++++-- .../Description/FunctionDescriptorProvider.cs | 4 +- .../FunctionConfigurationException.cs | 25 ++++ .../Host/FunctionMetadataManager.cs | 8 +- .../HostConfigurationException.cs | 25 ++++ .../ScriptConfigurationException.cs | 23 --- src/WebJobs.Script/ScriptConstants.cs | 2 + .../ScriptHostBuilderExtensions.cs | 7 +- .../ApplicationInsightsEndToEndTestsBase.cs | 3 + .../ApplicationInsightsTestFixture.cs | 1 - .../TestFunctionHost.cs | 27 ++-- .../TestScripts/CSharp/host.json | 3 +- .../TestScripts/DirectLoad/host.json | 1 + .../TestScripts/DotNet/host.json | 1 + .../TestScripts/FSharp/host.json | 1 + .../TestScripts/FunctionGeneration/host.json | 1 + .../TestScripts/ListenerExceptions/host.json | 1 + .../TestScripts/Node/host.json | 3 +- .../TestScripts/OutOfRange/host.json | 5 +- .../TestScripts/Proxies/host.json | 1 + .../HostConfigurationExceptionTests.cs | 62 ++++++++ .../HostJsonFileConfigurationSourceTests.cs | 134 ++++++++++++------ .../FunctionDescriptorProviderTests.cs | 4 +- .../FunctionMetadataManagerTests.cs | 4 +- test/WebJobs.Script.Tests/ScriptHostTests.cs | 6 +- 29 files changed, 345 insertions(+), 128 deletions(-) create mode 100644 src/WebJobs.Script/FunctionConfigurationException.cs create mode 100644 src/WebJobs.Script/HostConfigurationException.cs delete mode 100644 src/WebJobs.Script/ScriptConfigurationException.cs create mode 100644 test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/HostConfigurationExceptionTests.cs diff --git a/sample/host.json b/sample/host.json index 25b57084e9..61a30fe30a 100644 --- a/sample/host.json +++ b/sample/host.json @@ -1,4 +1,5 @@ { + "version": "2.0", "watchDirectories": [ "Shared", "Test" ], "healthMonitor": { "enabled": true, diff --git a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs index ff0eb3de3b..b8984e997f 100644 --- a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs +++ b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs @@ -18,7 +18,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.WebHost diff --git a/src/WebJobs.Script.WebHost/WebJobsApplicationBuilderExtension.cs b/src/WebJobs.Script.WebHost/WebJobsApplicationBuilderExtension.cs index c844bc7ff8..52316cec6b 100644 --- a/src/WebJobs.Script.WebHost/WebJobsApplicationBuilderExtension.cs +++ b/src/WebJobs.Script.WebHost/WebJobsApplicationBuilderExtension.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Azure.WebJobs.Script.WebHost.Features; using Microsoft.Azure.WebJobs.Script.WebHost.Middleware; using Microsoft.Extensions.DependencyInjection; @@ -46,7 +47,10 @@ public static IApplicationBuilder UseWebJobsScriptHost(this IApplicationBuilder builder.UseMiddleware(); builder.UseMiddleware(); builder.UseMiddleware(); - builder.UseMiddleware(); + builder.UseWhen(context => context.Features.Get() != null, config => + { + config.UseMiddleware(); + }); builder.UseMiddleware(); builder.UseMiddleware(); diff --git a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs index 5d3c85c8e9..47efbd5107 100644 --- a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs +++ b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs @@ -2,10 +2,8 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Globalization; using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Hosting; @@ -76,7 +74,7 @@ public async Task StartAsync(CancellationToken cancellationToken) } } - private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0) + private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, bool skipHostJsonConfiguration = false) { cancellationToken.ThrowIfCancellationRequested(); @@ -84,15 +82,19 @@ private async Task StartHostAsync(CancellationToken cancellationToken, int attem { bool isOffline = CheckAppOffline(); - _host = BuildHost(isOffline); + _host = BuildHost(isOffline, skipHostJsonConfiguration); LogInitialization(_host, isOffline, attemptCount, _hostStartCount++); await _host.StartAsync(cancellationToken); - LastError = null; + // This means we had an error on a previous load, so we want to keep the LastError around + if (!skipHostJsonConfiguration) + { + LastError = null; + } - if (!isOffline) + if (!isOffline && !skipHostJsonConfiguration) { State = ScriptHostState.Running; } @@ -122,12 +124,21 @@ private async Task StartHostAsync(CancellationToken cancellationToken, int attem cancellationToken.ThrowIfCancellationRequested(); - await Utility.DelayWithBackoffAsync(attemptCount, cancellationToken, min: TimeSpan.FromSeconds(1), max: TimeSpan.FromMinutes(2)) - .ContinueWith(t => - { - cancellationToken.ThrowIfCancellationRequested(); - return StartHostAsync(cancellationToken, attemptCount); - }); + if (exc is HostConfigurationException) + { + // Try starting the host without parsing host.json. This will start up a + // minimal host and allow the portal to see the error. Any modification will restart again. + Task ignore = StartHostAsync(cancellationToken, attemptCount, skipHostJsonConfiguration: true); + } + else + { + await Utility.DelayWithBackoffAsync(attemptCount, cancellationToken, min: TimeSpan.FromSeconds(1), max: TimeSpan.FromMinutes(2)) + .ContinueWith(t => + { + cancellationToken.ThrowIfCancellationRequested(); + return StartHostAsync(cancellationToken, attemptCount); + }); + } } } @@ -176,10 +187,19 @@ public async Task RestartHostAsync(CancellationToken cancellationToken) _logger.LogInformation("Script host restarted."); } - private IHost BuildHost(bool isOffline = false) + private IHost BuildHost(bool isOffline = false, bool skipHostJsonConfiguration = false) { - var builder = new HostBuilder() - .SetAzureFunctionsEnvironment() + var builder = new HostBuilder(); + + if (skipHostJsonConfiguration) + { + builder.ConfigureAppConfiguration((context, _) => + { + context.Properties[ScriptConstants.SkipHostJsonConfigurationKey] = true; + }); + } + + builder.SetAzureFunctionsEnvironment() .AddWebScriptHost(_rootServiceProvider, _rootScopeFactory, _applicationHostOptions.CurrentValue); if (isOffline) @@ -238,7 +258,7 @@ private async Task Orphan(IHost instance, ILogger logger, CancellationToken canc { try { - await instance?.StopAsync(cancellationToken); + await (instance?.StopAsync(cancellationToken) ?? Task.CompletedTask); } catch (Exception) { diff --git a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs index a2d55eb3c4..5301df8ab0 100644 --- a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs +++ b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs @@ -9,7 +9,6 @@ using Microsoft.Azure.WebJobs.Logging; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using static System.Environment; @@ -20,7 +19,7 @@ public class HostJsonFileConfigurationSource : IConfigurationSource { private readonly ILogger _logger; - public HostJsonFileConfigurationSource(ScriptApplicationHostOptions applicationHostOptions, ILoggerFactory loggerFactory) + public HostJsonFileConfigurationSource(ScriptApplicationHostOptions applicationHostOptions, IEnvironment environment, ILoggerFactory loggerFactory) { if (loggerFactory == null) { @@ -28,11 +27,14 @@ public HostJsonFileConfigurationSource(ScriptApplicationHostOptions applicationH } HostOptions = applicationHostOptions; + Environment = environment; _logger = loggerFactory.CreateLogger(LogCategories.Startup); } public ScriptApplicationHostOptions HostOptions { get; } + public IEnvironment Environment { get; } + public IConfigurationProvider Build(IConfigurationBuilder builder) { return new HostJsonFileConfigurationProvider(this, _logger); @@ -42,8 +44,8 @@ private class HostJsonFileConfigurationProvider : ConfigurationProvider { private static readonly string[] WellKnownHostJsonProperties = new[] { - "id", "functionTimeout", "functions", "http", "watchDirectories", "queues", "serviceBus", - "eventHub", "tracing", "singleton", "logger", "aggregator", "applicationInsights", "healthMonitor" + "version", "id", "functionTimeout", "functions", "http", "watchDirectories", "queues", "serviceBus", + "eventHub", "singleton", "logging", "aggregator", "healthMonitor" }; private readonly HostJsonFileConfigurationSource _configurationSource; @@ -133,6 +135,7 @@ private JObject LoadHostConfigurationFile() string hostFilePath = Path.Combine(options.ScriptPath, ScriptConstants.HostMetadataFileName); string readingFileMessage = string.Format(CultureInfo.InvariantCulture, "Reading host configuration file '{0}'", hostFilePath); JObject hostConfigObject = LoadHostConfig(hostFilePath); + InitializeHostConfig(hostFilePath, hostConfigObject); string sanitizedJson = SanitizeHostJson(hostConfigObject); string readFileMessage = $"Host configuration file read:{NewLine}{sanitizedJson}"; @@ -190,7 +193,24 @@ private JObject LoadHostConfigurationFile() return hostConfigObject; } - internal static JObject LoadHostConfig(string configFilePath) + private void InitializeHostConfig(string hostJsonPath, JObject hostConfigObject) + { + // If the object is empty, initialize it to include the version and write the file. + if (!hostConfigObject.HasValues) + { + _logger.LogInformation($"Empty host configuration file found. Creating a default {ScriptConstants.HostMetadataFileName} file."); + + hostConfigObject = GetDefaultHostConfigObject(); + TryWriteHostJson(hostJsonPath, hostConfigObject); + } + + if (hostConfigObject["version"]?.Value() != "2.0") + { + throw new HostConfigurationException($"The {ScriptConstants.HostMetadataFileName} file is missing the required 'version' property. See https://aka.ms/functions-hostjson for steps to migrate the configuration file."); + } + } + + internal JObject LoadHostConfig(string configFilePath) { JObject hostConfigObject; try @@ -205,14 +225,39 @@ internal static JObject LoadHostConfig(string configFilePath) catch (Exception ex) when (ex is FileNotFoundException || ex is DirectoryNotFoundException) { // if no file exists we default the config - // TODO: DI (FACAVAL) Log - //logger.LogInformation("No host configuration file found. Using default."); - hostConfigObject = new JObject(); + _logger.LogInformation($"No host configuration file found. Creating a default {ScriptConstants.HostMetadataFileName} file."); + + hostConfigObject = GetDefaultHostConfigObject(); + TryWriteHostJson(configFilePath, hostConfigObject); } return hostConfigObject; } + private static JObject GetDefaultHostConfigObject() + { + return new JObject { { "version", "2.0" } }; + } + + private void TryWriteHostJson(string filePath, JObject content) + { + if (!_configurationSource.Environment.FileSystemIsReadOnly()) + { + try + { + File.WriteAllText(filePath, content.ToString(Formatting.Indented)); + } + catch + { + _logger.LogInformation($"Failed to create {ScriptConstants.HostMetadataFileName} file. Host execution will continue."); + } + } + else + { + _logger.LogInformation($"File system is read-only. Skipping {ScriptConstants.HostMetadataFileName} creation."); + } + } + internal static string SanitizeHostJson(JObject hostJsonObject) { JObject sanitizedObject = new JObject(); diff --git a/src/WebJobs.Script/Description/FunctionDescriptorProvider.cs b/src/WebJobs.Script/Description/FunctionDescriptorProvider.cs index 6afcc683f8..c515a9a455 100644 --- a/src/WebJobs.Script/Description/FunctionDescriptorProvider.cs +++ b/src/WebJobs.Script/Description/FunctionDescriptorProvider.cs @@ -85,7 +85,7 @@ public void VerifyResolvedBindings(FunctionMetadata functionMetadata, IEnumerabl if (unresolvedBindings.Any()) { string allUnresolvedBindings = string.Join(", ", unresolvedBindings); - throw new ScriptConfigurationException($"The binding type(s) '{allUnresolvedBindings}' are not registered. " + + throw new FunctionConfigurationException($"The binding type(s) '{allUnresolvedBindings}' are not registered. " + $"Please ensure the type is correct and the binding extension is installed."); } } @@ -135,7 +135,7 @@ protected virtual ParameterDescriptor CreateTriggerParameter(BindingMetadata tri } else { - throw new ScriptConfigurationException($"The binding type '{triggerMetadata.Type}' is not registered. " + + throw new FunctionConfigurationException($"The binding type '{triggerMetadata.Type}' is not registered. " + $"Please ensure the type is correct and the binding extension is installed."); } diff --git a/src/WebJobs.Script/FunctionConfigurationException.cs b/src/WebJobs.Script/FunctionConfigurationException.cs new file mode 100644 index 0000000000..19163937c9 --- /dev/null +++ b/src/WebJobs.Script/FunctionConfigurationException.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.WebJobs.Script +{ + /// + /// An exception that indicates an issue with a function. These exceptions will be caught and + /// logged, but not cause a host to restart. + /// + [Serializable] + public class FunctionConfigurationException : Exception + { + public FunctionConfigurationException() { } + + public FunctionConfigurationException(string message) : base(message) { } + + public FunctionConfigurationException(string message, Exception inner) : base(message, inner) { } + + protected FunctionConfigurationException( + System.Runtime.Serialization.SerializationInfo info, + System.Runtime.Serialization.StreamingContext context) : base(info, context) { } + } +} diff --git a/src/WebJobs.Script/Host/FunctionMetadataManager.cs b/src/WebJobs.Script/Host/FunctionMetadataManager.cs index 5f8ea9333c..be34d6102c 100644 --- a/src/WebJobs.Script/Host/FunctionMetadataManager.cs +++ b/src/WebJobs.Script/Host/FunctionMetadataManager.cs @@ -145,7 +145,7 @@ internal static bool TryParseFunctionMetadata(string functionName, JObject funct { functionMetadata.ScriptFile = DeterminePrimaryScriptFile(functionConfig, scriptDirectory, fileSystem); } - catch (ScriptConfigurationException exc) + catch (FunctionConfigurationException exc) { error = exc.Message; return false; @@ -216,7 +216,7 @@ internal static string DeterminePrimaryScriptFile(JObject functionConfig, string string scriptPath = fileSystem.Path.Combine(scriptDirectory, scriptFile); if (!fileSystem.File.Exists(scriptPath)) { - throw new ScriptConfigurationException("Invalid script file name configuration. The 'scriptFile' property is set to a file that does not exist."); + throw new FunctionConfigurationException("Invalid script file name configuration. The 'scriptFile' property is set to a file that does not exist."); } functionPrimary = scriptPath; @@ -229,7 +229,7 @@ internal static string DeterminePrimaryScriptFile(JObject functionConfig, string if (functionFiles.Length == 0) { - throw new ScriptConfigurationException("No function script files present."); + throw new FunctionConfigurationException("No function script files present."); } if (functionFiles.Length == 1) @@ -249,7 +249,7 @@ internal static string DeterminePrimaryScriptFile(JObject functionConfig, string if (string.IsNullOrEmpty(functionPrimary)) { - throw new ScriptConfigurationException("Unable to determine the primary function script. Try renaming your entry point script to 'run' (or 'index' in the case of Node), " + + throw new FunctionConfigurationException("Unable to determine the primary function script. Try renaming your entry point script to 'run' (or 'index' in the case of Node), " + "or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata."); } diff --git a/src/WebJobs.Script/HostConfigurationException.cs b/src/WebJobs.Script/HostConfigurationException.cs new file mode 100644 index 0000000000..a637c098b2 --- /dev/null +++ b/src/WebJobs.Script/HostConfigurationException.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.WebJobs.Script +{ + /// + /// An exception that indicates an issue configuring a ScriptHost. This will + /// prevent the host from starting. + /// + [Serializable] + public class HostConfigurationException : Exception + { + public HostConfigurationException() { } + + public HostConfigurationException(string message) : base(message) { } + + public HostConfigurationException(string message, Exception inner) : base(message, inner) { } + + protected HostConfigurationException( + System.Runtime.Serialization.SerializationInfo info, + System.Runtime.Serialization.StreamingContext context) : base(info, context) { } + } +} diff --git a/src/WebJobs.Script/ScriptConfigurationException.cs b/src/WebJobs.Script/ScriptConfigurationException.cs deleted file mode 100644 index 0a9bde0c3c..0000000000 --- a/src/WebJobs.Script/ScriptConfigurationException.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace Microsoft.Azure.WebJobs.Script -{ - [Serializable] - public class ScriptConfigurationException : Exception - { - public ScriptConfigurationException() { } - - public ScriptConfigurationException(string message) : base(message) { } - - public ScriptConfigurationException(string message, Exception inner) : base(message, inner) { } - - protected ScriptConfigurationException( - System.Runtime.Serialization.SerializationInfo info, - System.Runtime.Serialization.StreamingContext context) : base(info, context) { } - } -} diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index 8996723976..0d24b052cc 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -50,6 +50,8 @@ public static class ScriptConstants public const string LogCategoryWorker = "Worker"; public const string LogCategoryMigration = "Host.Migration"; + public const string SkipHostJsonConfigurationKey = "MS_SkipHostJsonConfiguration"; + // Define all system parameters we inject with a prefix to avoid collisions // with user parameters public const string SystemTriggerParameterName = "_triggerValue"; diff --git a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs index 3fa1c42aa4..93b0bd7d09 100644 --- a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs +++ b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs @@ -66,9 +66,12 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder, ScriptApplic ConfigureApplicationInsights(context, loggingBuilder); }) - .ConfigureAppConfiguration(c => + .ConfigureAppConfiguration((context, configBuilder) => { - c.Add(new HostJsonFileConfigurationSource(applicationOptions, loggerFactory)); + if (!context.Properties.ContainsKey(ScriptConstants.SkipHostJsonConfigurationKey)) + { + configBuilder.Add(new HostJsonFileConfigurationSource(applicationOptions, SystemEnvironment.Instance, loggerFactory)); + } }); // WebJobs configuration diff --git a/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsEndToEndTestsBase.cs b/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsEndToEndTestsBase.cs index 83d4468b53..1e7257cf23 100644 --- a/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsEndToEndTestsBase.cs +++ b/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsEndToEndTestsBase.cs @@ -248,6 +248,9 @@ await TestHelpers.Await(() => // TODO: Remove this once the issue https://github.com/Azure/azure-functions-nodejs-worker/issues/98 is resolved traces = traces.Where(t => !t.Message.Contains("[DEP0005]")).ToArray(); + // We may have any number of "Host Status" calls as we wait for startup. Let's ignore them. + traces = traces.Where(t => !t.Message.StartsWith("Host Status")).ToArray(); + Assert.True(traces.Length == expectedCount, $"Expected {expectedCount} messages, but found {traces.Length}. Actual logs:{Environment.NewLine}{string.Join(Environment.NewLine, traces.Select(t => t.Message))}"); ValidateTrace(traces[0], "A function whitelist has been specified", LogCategories.Startup); diff --git a/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsTestFixture.cs b/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsTestFixture.cs index 2df33a5b62..2ab6c38d46 100644 --- a/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsTestFixture.cs +++ b/test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsTestFixture.cs @@ -49,7 +49,6 @@ public ApplicationInsightsTestFixture(string scriptRoot, string testId) }); HttpClient = TestHost.HttpClient; - HttpClient.BaseAddress = new Uri("https://localhost/"); TestHelpers.WaitForWebHost(HttpClient); } diff --git a/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs b/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs index c7906b27ef..b3e318aab9 100644 --- a/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs +++ b/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs @@ -6,7 +6,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Net; using System.Net.Http; using System.Text; using System.Threading; @@ -81,9 +80,9 @@ public TestFunctionHost(string appRoot, HttpClient.BaseAddress = new Uri("https://localhost/"); var manager = _testServer.Host.Services.GetService(); - manager.DelayUntilHostReady().GetAwaiter().GetResult(); - _hostService = manager as WebJobsScriptHostService; + + StartAsync().GetAwaiter().GetResult(); } public IServiceProvider JobHostServices => _hostService.Services; @@ -110,16 +109,16 @@ public async Task GetFunctionSecretAsync(string functionName) return secrets.First().Value; } - public async Task StartAsync() + private async Task StartAsync() { bool running = false; while (!running) { - running = await IsHostRunning(HttpClient); + running = await IsHostStarted(HttpClient); if (!running) { - await Task.Delay(500); + await Task.Delay(50); } } } @@ -203,20 +202,10 @@ public void Dispose() _testServer.Dispose(); } - private async Task IsHostRunning(HttpClient client) + private async Task IsHostStarted(HttpClient client) { - HostSecretsInfo secrets = await SecretManager.GetHostSecretsAsync(); - - // Workaround for https://github.com/Azure/azure-functions-host/issues/2397 as the base URL - // doesn't currently start the host. - // Note: the master key "1234" is from the TestSecretManager. - using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, $"/admin/functions/dummyName/status?code={secrets.MasterKey}")) - { - using (HttpResponseMessage response = await client.SendAsync(request)) - { - return response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NotFound; - } - } + HostStatus status = await GetHostStatusAsync(); + return status.State == $"{ScriptHostState.Running}" || status.State == $"{ScriptHostState.Error}"; } private class UpdateContentLengthHandler : DelegatingHandler diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/CSharp/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/CSharp/host.json index 09b5ede1da..01f5ab53b9 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/CSharp/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/CSharp/host.json @@ -1,3 +1,4 @@ { - "id": "function-tests-csharp" + "version": "2.0", + "id": "function-tests-csharp" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/DirectLoad/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/DirectLoad/host.json index d55b423add..26c53d4c5b 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/DirectLoad/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/DirectLoad/host.json @@ -1,3 +1,4 @@ { + "version": "2.0", "id": "function-tests-dotnet-direct" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/DotNet/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/DotNet/host.json index 64633b13fd..c7f0dbf29c 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/DotNet/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/DotNet/host.json @@ -1,3 +1,4 @@ { + "version": "2.0", "id": "function-tests-dotnet" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/FSharp/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/FSharp/host.json index adb6cbd5a6..b59c6ab709 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/FSharp/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/FSharp/host.json @@ -1,3 +1,4 @@ { + "version": "2.0", "id": "function-tests-fsharp" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/FunctionGeneration/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/FunctionGeneration/host.json index cf502e147a..79cafb5a27 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/FunctionGeneration/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/FunctionGeneration/host.json @@ -1,3 +1,4 @@ { + "version": "2.0", "id": "function-tests-functiongen" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/ListenerExceptions/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/ListenerExceptions/host.json index c411d27976..225c71528b 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/ListenerExceptions/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/ListenerExceptions/host.json @@ -1,3 +1,4 @@ { + "version": "2.0", "id": "function-listener-failues" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/Node/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/Node/host.json index 35faaa7a5b..f3ad82ffca 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/Node/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/Node/host.json @@ -1,3 +1,4 @@ { - "id": "function-tests-node" + "version": "2.0", + "id": "function-tests-node" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfRange/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfRange/host.json index 09c2f5079f..86401242b7 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfRange/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfRange/host.json @@ -1,6 +1,7 @@ -{ +{ + "version": "2.0", "logger": { - "aggregator": { + "aggregator": { "flushTimeout": "00:30:00" } } diff --git a/test/WebJobs.Script.Tests.Integration/TestScripts/Proxies/host.json b/test/WebJobs.Script.Tests.Integration/TestScripts/Proxies/host.json index 7a0f1d1abd..88c5047971 100644 --- a/test/WebJobs.Script.Tests.Integration/TestScripts/Proxies/host.json +++ b/test/WebJobs.Script.Tests.Integration/TestScripts/Proxies/host.json @@ -1,3 +1,4 @@ { + "version": "2.0", "id": "function-tests-proxies" } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/HostConfigurationExceptionTests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/HostConfigurationExceptionTests.cs new file mode 100644 index 0000000000..82dac6c34b --- /dev/null +++ b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/HostConfigurationExceptionTests.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Script.WebHost.Models; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.WebHostEndToEnd +{ + public class HostConfigurationExceptionTests : IDisposable + { + private readonly string _hostPath; + + public HostConfigurationExceptionTests() + { + _hostPath = Path.Combine(Path.GetTempPath(), "Functions", "HostConfigurationExceptionTests"); + Directory.CreateDirectory(_hostPath); + } + + [Fact] + public async Task HostStatusReturns_IfHostJsonError() + { + string hostJsonPath = Path.Combine(_hostPath, ScriptConstants.HostMetadataFileName); + + // Simulate a non-empty file without a 'version' + JObject hostConfig = JObject.FromObject(new + { + functionTimeout = TimeSpan.FromSeconds(30) + }); + + await File.WriteAllTextAsync(hostJsonPath, hostConfig.ToString()); + + var host = new TestFunctionHost(_hostPath, _ => { }); + + // Ping the status endpoint to ensure we see the exception + HostStatus status = await host.GetHostStatusAsync(); + Assert.Equal("Error", status.State); + Assert.Equal("Microsoft.Azure.WebJobs.Script: The host.json file is missing the required 'version' property. See https://aka.ms/functions-hostjson for steps to migrate the configuration file.", status.Errors.Single()); + + // Now update the file and make sure it auto-restarts. + hostConfig["version"] = "2.0"; + await File.WriteAllTextAsync(hostJsonPath, hostConfig.ToString()); + + await TestHelpers.Await(async () => + { + status = await host.GetHostStatusAsync(); + return status.State == $"{ScriptHostState.Running}"; + }); + + Assert.Null(status.Errors); + } + + public void Dispose() + { + Directory.Delete(_hostPath, true); + } + } +} diff --git a/test/WebJobs.Script.Tests/Configuration/HostJsonFileConfigurationSourceTests.cs b/test/WebJobs.Script.Tests/Configuration/HostJsonFileConfigurationSourceTests.cs index 8d140468f2..9ba3a53b56 100644 --- a/test/WebJobs.Script.Tests/Configuration/HostJsonFileConfigurationSourceTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/HostJsonFileConfigurationSourceTests.cs @@ -2,13 +2,13 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using Microsoft.Azure.WebJobs.Logging; using Microsoft.Azure.WebJobs.Script.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; using Newtonsoft.Json.Linq; using Xunit; @@ -17,25 +17,102 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Configuration { public class HostJsonFileConfigurationSourceTests { - [Fact] - public void Initialize_Sanitizes_HostJsonLog() - { - var loggerFactory = new LoggerFactory(); - TestLoggerProvider loggerProvider = new TestLoggerProvider(); - loggerFactory.AddProvider(loggerProvider); + private readonly string _defaultHostJson = "{\r\n \"version\": \"2.0\"\r\n}"; + private readonly ScriptApplicationHostOptions _options; + private readonly string _hostJsonFile; + private readonly TestLoggerProvider _loggerProvider = new TestLoggerProvider(); + public HostJsonFileConfigurationSourceTests() + { string rootPath = Path.Combine(Environment.CurrentDirectory, "ScriptHostTests"); + if (!Directory.Exists(rootPath)) { Directory.CreateDirectory(rootPath); } + _options = new ScriptApplicationHostOptions + { + ScriptPath = rootPath + }; + + // delete any existing host.json + _hostJsonFile = Path.Combine(rootPath, "host.json"); + if (File.Exists(_hostJsonFile)) + { + File.Delete(_hostJsonFile); + } + } + + [Fact] + public void MissingHostJson_CreatesDefaultFile() + { + Assert.False(File.Exists(_hostJsonFile)); + BuildHostJsonConfiguration(); + + Assert.Equal(_defaultHostJson, File.ReadAllText(_hostJsonFile)); + + var log = _loggerProvider.GetAllLogMessages().Single(l => l.FormattedMessage == "No host configuration file found. Creating a default host.json file."); + Assert.Equal(LogLevel.Information, log.Level); + } + + [Theory] + [InlineData("{}")] + [InlineData("{\r\n}")] + public void EmptyHostJson_CreatesDefaultFile(string json) + { + File.WriteAllText(_hostJsonFile, json); + Assert.True(File.Exists(_hostJsonFile)); + BuildHostJsonConfiguration(); + + Assert.Equal(_defaultHostJson, File.ReadAllText(_hostJsonFile)); + + var log = _loggerProvider.GetAllLogMessages().Single(l => l.FormattedMessage == "Empty host configuration file found. Creating a default host.json file."); + Assert.Equal(LogLevel.Information, log.Level); + } + + [Fact] + public void MissingVersion_ThrowsException() + { + string hostJsonContent = @" + { + 'functions': [ 'FunctionA', 'FunctionB' ] + }"; + + File.WriteAllText(_hostJsonFile, hostJsonContent); + Assert.True(File.Exists(_hostJsonFile)); + + var ex = Assert.Throws(() => BuildHostJsonConfiguration()); + Assert.StartsWith("The host.json file is missing the required 'version' property.", ex.Message); + } + + [Fact] + public void ReadOnlyFileSystem_SkipsDefaultHostJsonCreation() + { + Assert.False(File.Exists(_hostJsonFile)); + + var environment = new TestEnvironment(new Dictionary + { + { EnvironmentSettingNames.AzureWebsiteZipDeployment, "1" } + }); + + IConfiguration config = BuildHostJsonConfiguration(environment); + Assert.Equal(config["AzureFunctionsJobHost:version"], "2.0"); + + var log = _loggerProvider.GetAllLogMessages().Single(l => l.FormattedMessage == "No host configuration file found. Creating a default host.json file."); + Assert.Equal(LogLevel.Information, log.Level); + } + + [Fact] + public void Initialize_Sanitizes_HostJsonLog() + { // Turn off all logging. We shouldn't see any output. string hostJsonContent = @" { + 'version': '2.0', 'functionTimeout': '00:05:00', 'functions': [ 'FunctionA', 'FunctionB' ], - 'logger': { + 'logging': { 'categoryFilter': { 'defaultLevel': 'Information' } @@ -45,25 +122,16 @@ public void Initialize_Sanitizes_HostJsonLog() } }"; - File.WriteAllText(Path.Combine(rootPath, "host.json"), hostJsonContent); + File.WriteAllText(_hostJsonFile, hostJsonContent); - var webHostOptions = new ScriptApplicationHostOptions - { - ScriptPath = rootPath - }; - - var configSource = new HostJsonFileConfigurationSource(webHostOptions, loggerFactory); - - var configurationBuilder = new ConfigurationBuilder(); - IConfigurationProvider provider = configSource.Build(configurationBuilder); - - provider.Load(); + BuildHostJsonConfiguration(); string hostJsonSanitized = @" { + 'version': '2.0', 'functionTimeout': '00:05:00', 'functions': [ 'FunctionA', 'FunctionB' ], - 'logger': { + 'logging': { 'categoryFilter': { 'defaultLevel': 'Information' } @@ -73,33 +141,19 @@ public void Initialize_Sanitizes_HostJsonLog() // for formatting var hostJson = JObject.Parse(hostJsonSanitized); - var logger = loggerProvider.CreatedLoggers.Single(l => l.Category == LogCategories.Startup); + var logger = _loggerProvider.CreatedLoggers.Single(l => l.Category == LogCategories.Startup); var logMessage = logger.GetLogMessages().Single(l => l.FormattedMessage.StartsWith("Host configuration file read")).FormattedMessage; Assert.Equal($"Host configuration file read:{Environment.NewLine}{hostJson}", logMessage); } - private IConfiguration GetConfiguration(JObject hostConfiguration) + private IConfiguration BuildHostJsonConfiguration(IEnvironment environment = null) { - string rootPath = Path.Combine(Environment.CurrentDirectory, "ScriptHostTests"); - - string hostJsonContent = hostConfiguration.ToString(); - File.WriteAllText(Path.Combine(rootPath, "host.json"), hostJsonContent); + environment = environment ?? new TestEnvironment(); var loggerFactory = new LoggerFactory(); - TestLoggerProvider loggerProvider = new TestLoggerProvider(); - loggerFactory.AddProvider(loggerProvider); - - if (!Directory.Exists(rootPath)) - { - Directory.CreateDirectory(rootPath); - } - - var webHostOptions = new ScriptApplicationHostOptions - { - ScriptPath = rootPath - }; + loggerFactory.AddProvider(_loggerProvider); - var configSource = new HostJsonFileConfigurationSource(webHostOptions, loggerFactory); + var configSource = new HostJsonFileConfigurationSource(_options, environment, loggerFactory); var configurationBuilder = new ConfigurationBuilder() .Add(configSource); diff --git a/test/WebJobs.Script.Tests/Description/FunctionDescriptorProviderTests.cs b/test/WebJobs.Script.Tests/Description/FunctionDescriptorProviderTests.cs index 8fd82b8136..a7a76c6dbc 100644 --- a/test/WebJobs.Script.Tests/Description/FunctionDescriptorProviderTests.cs +++ b/test/WebJobs.Script.Tests/Description/FunctionDescriptorProviderTests.cs @@ -97,7 +97,7 @@ public void VerifyResolvedBindings_WithNoBindingMatch_ThrowsExpectedException() functionMetadata.Bindings.Add(triggerMetadata); functionMetadata.Bindings.Add(bindingMetadata); - var ex = Assert.Throws(() => + var ex = Assert.Throws(() => { _provider.TryCreate(functionMetadata, out FunctionDescriptor descriptor); }); @@ -134,7 +134,7 @@ public void CreateTriggerParameter_WithNoBindingMatch_ThrowsExpectedException() functionMetadata.Bindings.Add(metadata); - var ex = Assert.Throws(() => + var ex = Assert.Throws(() => { _provider.TryCreate(functionMetadata, out FunctionDescriptor descriptor); }); diff --git a/test/WebJobs.Script.Tests/FunctionMetadataManagerTests.cs b/test/WebJobs.Script.Tests/FunctionMetadataManagerTests.cs index 39a3346d42..ceaecf3ae0 100644 --- a/test/WebJobs.Script.Tests/FunctionMetadataManagerTests.cs +++ b/test/WebJobs.Script.Tests/FunctionMetadataManagerTests.cs @@ -71,7 +71,7 @@ public void DeterminePrimaryScriptFile_MultipleFiles_NoClearPrimary_ReturnsNull( { @"c:\functions\test.txt", new MockFileData(string.Empty) } }; var fileSystem = new MockFileSystem(files); - Assert.Throws(() => FunctionMetadataManager.DeterminePrimaryScriptFile(functionConfig, @"c:\functions", fileSystem)); + Assert.Throws(() => FunctionMetadataManager.DeterminePrimaryScriptFile(functionConfig, @"c:\functions", fileSystem)); } [Fact] @@ -82,7 +82,7 @@ public void DeterminePrimaryScriptFile_NoFiles_ReturnsNull() var fileSystem = new MockFileSystem(); fileSystem.AddDirectory(@"c:\functions"); - Assert.Throws(() => FunctionMetadataManager.DeterminePrimaryScriptFile(functionConfig, @"c:\functions", fileSystem)); + Assert.Throws(() => FunctionMetadataManager.DeterminePrimaryScriptFile(functionConfig, @"c:\functions", fileSystem)); } [Fact] diff --git a/test/WebJobs.Script.Tests/ScriptHostTests.cs b/test/WebJobs.Script.Tests/ScriptHostTests.cs index 24747cf57e..3a22b2d5a1 100644 --- a/test/WebJobs.Script.Tests/ScriptHostTests.cs +++ b/test/WebJobs.Script.Tests/ScriptHostTests.cs @@ -15,7 +15,6 @@ using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Eventing; -using Microsoft.Azure.WebJobs.Script.WebHost; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -294,6 +293,7 @@ public async Task Initialize_InvalidFunctionNames_DoesNotCreateFunctionAndLogsFa Directory.CreateDirectory(invalidFunctionNamePath); JObject config = new JObject(); + config["version"] = "2.0"; config["id"] = ID; File.WriteAllText(Path.Combine(rootPath, ScriptConstants.HostMetadataFileName), config.ToString()); @@ -616,8 +616,7 @@ public void TryGetFunctionFromException_FunctionMatch() var exception = new InvalidOperationException(stack); // no match - empty functions - FunctionDescriptor functionResult = null; - bool result = ScriptHost.TryGetFunctionFromException(functions, exception, out functionResult); + bool result = ScriptHost.TryGetFunctionFromException(functions, exception, out FunctionDescriptor functionResult); Assert.False(result); Assert.Null(functionResult); @@ -884,6 +883,7 @@ public async Task Initialize_LogsWarningForExplicitlySetHostId() // Set id in the host.json string hostJsonContent = @" { + 'version': '2.0', 'id': 'foobar' }"; File.WriteAllText(Path.Combine(rootPath, "host.json"), hostJsonContent);