From 79017d99f346ebd5b142aac85bc0723d2e3cd06b Mon Sep 17 00:00:00 2001 From: lingtoh Date: Tue, 24 Apr 2018 16:33:31 -0700 Subject: [PATCH] Read storage connection in host.json for DurableFunctions --- schemas/json/host.json | 19 +++++- .../Management/WebFunctionsManager.cs | 67 +++++++++++++------ .../Managment/WebFunctionsManagerTests.cs | 29 +++++++- 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/schemas/json/host.json b/schemas/json/host.json index dc45a7d173..15c9473947 100644 --- a/schemas/json/host.json +++ b/schemas/json/host.json @@ -282,6 +282,23 @@ } } } + }, + + "durableTask": { + "description": "Configuration settings for 'orchestration'/'activity' triggers.", + "type": "object", + + "properties": { + "hubName": { + "description": "The logical container for Azure Storage resources that are used for orchestrations.", + "type": "string", + "default": "DurableFunctionsHub" + }, + + "azureStorageConnectionStringName": { + "description": "An app setting (or environment variable) with the storage connection string to be used by the orchestration/activity trigger.", + "type": "string" + } + } } - } } diff --git a/src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs b/src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs index c6cf8a1909..85963e6f5e 100755 --- a/src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs +++ b/src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs @@ -13,7 +13,6 @@ using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Management.Models; using Microsoft.Azure.WebJobs.Script.WebHost.Extensions; -using Microsoft.Azure.WebJobs.Script.WebHost.Helpers; using Microsoft.Azure.WebJobs.Script.WebHost.Security; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -23,6 +22,12 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost.Management { public class WebFunctionsManager : IWebFunctionsManager { + private const string HubName = "HubName"; + private const string TaskHubName = "taskHubName"; + private const string Connection = "connection"; + private const string DurableTaskStorageConnectionName = "azureStorageConnectionStringName"; + private const string DurableTask = "durableTask"; + private readonly ScriptHostConfiguration _config; private readonly ILogger _logger; private readonly HttpClient _client; @@ -167,7 +172,7 @@ await functionMetadata /// (success, error) public async Task<(bool success, string error)> TrySyncTriggers() { - var durableTaskHubName = await GetDurableTaskHubName(); + var durableTaskConfig = await ReadDurableTaskConfig(); var functionsTriggers = (await GetFunctionsMetadata() .Select(f => f.ToFunctionTrigger(_config)) .WhenAll()) @@ -176,11 +181,19 @@ await functionMetadata { // if we have a durableTask hub name and the function trigger is either orchestrationTrigger OR activityTrigger, // add a property "taskHubName" with durable task hub name. - if (durableTaskHubName != null + if (durableTaskConfig.Any() && (t["type"]?.ToString().Equals("orchestrationTrigger", StringComparison.OrdinalIgnoreCase) == true || t["type"]?.ToString().Equals("activityTrigger", StringComparison.OrdinalIgnoreCase) == true)) { - t["taskHubName"] = durableTaskHubName; + if (durableTaskConfig.ContainsKey(HubName)) + { + t[TaskHubName] = durableTaskConfig[HubName]; + } + + if (durableTaskConfig.ContainsKey(Connection)) + { + t[Connection] = durableTaskConfig[Connection]; + } } return t; }); @@ -225,22 +238,48 @@ private IEnumerable GetFunctionsMetadata() .ReadFunctionsMetadata(FileUtility.EnumerateDirectories(_config.RootScriptPath), _logger, new Dictionary>(), fileSystem: FileUtility.Instance); } - private async Task GetDurableTaskHubName() + private async Task> ReadDurableTaskConfig() { string hostJsonPath = Path.Combine(_config.RootScriptPath, ScriptConstants.HostMetadataFileName); + var config = new Dictionary(); if (FileUtility.FileExists(hostJsonPath)) { + var hostJson = JObject.Parse(await FileUtility.ReadAsync(hostJsonPath)); + JToken durableTaskValue; + + // we will allow case insensitivity given it is likely user hand edited + // see https://github.com/Azure/azure-functions-durable-extension/issues/111 + // // We're looking for {VALUE} // { // "durableTask": { - // "HubName": "{VALUE}" + // "hubName": "{VALUE}", + // "azureStorageConnectionStringName": "{VALUE}" // } // } - var hostJson = JsonConvert.DeserializeObject(await FileUtility.ReadAsync(hostJsonPath)); - return hostJson?.DurableTask?.HubName; + if (hostJson.TryGetValue(DurableTask, StringComparison.OrdinalIgnoreCase, out durableTaskValue) && durableTaskValue != null) + { + try + { + var kvp = (JObject)durableTaskValue; + if (kvp.TryGetValue(HubName, StringComparison.OrdinalIgnoreCase, out JToken nameValue) && nameValue != null) + { + config.Add(HubName, nameValue.ToString()); + } + + if (kvp.TryGetValue(DurableTaskStorageConnectionName, StringComparison.OrdinalIgnoreCase, out nameValue) && nameValue != null) + { + config.Add(Connection, nameValue.ToString()); + } + } + catch (Exception) + { + throw new InvalidDataException("Invalid host.json configuration for 'durableTask'."); + } + } } - return null; + return config; } private void DeleteFunctionArtifacts(FunctionMetadataResponse function) @@ -252,15 +291,5 @@ private void DeleteFunctionArtifacts(FunctionMetadataResponse function) FileUtility.DeleteFileSafe(testDataPath); } } - - private class HostJsonModel - { - public DurableTaskHostModel DurableTask { get; set; } - } - - private class DurableTaskHostModel - { - public string HubName { get; set; } - } } } diff --git a/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs b/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs index 1d6cc7060f..62f6bad5c8 100644 --- a/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs +++ b/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs @@ -23,7 +23,8 @@ public async Task VerifyDurableTaskHubNameIsAdded() { // Setup const string expectedSyncTriggersPayload = "[{\"authLevel\":\"anonymous\",\"type\":\"httpTrigger\",\"direction\":\"in\",\"name\":\"req\",\"functionName\":\"function1\"}," + - "{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"\",\"functionName\":\"function2\",\"taskHubName\":\"TestHubValue\"}]"; + "{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"DurableStorage\",\"functionName\":\"function2\",\"taskHubName\":\"TestHubValue\"}," + + "{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"DurableStorage\",\"functionName\":\"function3\",\"taskHubName\":\"TestHubValue\"}]"; var settings = CreateWebSettings(); var fileSystem = CreateFileSystem(settings.ScriptPath); var loggerFactory = MockNullLogerFactory.CreateLoggerFactory(); @@ -75,7 +76,7 @@ private static IFileSystem CreateFileSystem(string rootPath) fileSystem.SetupGet(f => f.File).Returns(fileBase.Object); fileBase.Setup(f => f.Exists(Path.Combine(rootPath, "host.json"))).Returns(true); - var hostJson = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""durableTask"": { ""HubName"": ""TestHubValue"" }}")); + var hostJson = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""durableTask"": { ""HubName"": ""TestHubValue"", ""azureStorageConnectionStringName"": ""DurableStorage"" }}")); hostJson.Position = 0; fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"host.json"), It.IsAny(), It.IsAny(), It.IsAny())).Returns(hostJson); @@ -85,7 +86,8 @@ private static IFileSystem CreateFileSystem(string rootPath) .Returns(new[] { @"x:\root\function1", - @"x:\root\function2" + @"x:\root\function2", + @"x:\root\function3" }); var function1 = @"{ @@ -118,10 +120,26 @@ private static IFileSystem CreateFileSystem(string rootPath) } ] }"; + + var function3 = @"{ + ""disabled"": false, + ""scriptFile"": ""main.js"", + ""bindings"": [ + { + ""name"": ""myQueueItem"", + ""type"": ""activityTrigger"", + ""direction"": ""in"", + ""queueName"": ""myqueue-items"", + ""connection"": """" + } + ] +}"; var function1Stream = new MemoryStream(Encoding.UTF8.GetBytes(function1)); function1Stream.Position = 0; var function2Stream = new MemoryStream(Encoding.UTF8.GetBytes(function2)); function2Stream.Position = 0; + var function3Stream = new MemoryStream(Encoding.UTF8.GetBytes(function3)); + function3Stream.Position = 0; fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function1\function.json"))).Returns(true); fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function1\main.py"))).Returns(true); fileBase.Setup(f => f.ReadAllText(Path.Combine(rootPath, @"function1\function.json"))).Returns(function1); @@ -132,6 +150,11 @@ private static IFileSystem CreateFileSystem(string rootPath) fileBase.Setup(f => f.ReadAllText(Path.Combine(rootPath, @"function2\function.json"))).Returns(function2); fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"function2\function.json"), It.IsAny(), It.IsAny(), It.IsAny())).Returns(function2Stream); + fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function3\function.json"))).Returns(true); + fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function3\main.js"))).Returns(true); + fileBase.Setup(f => f.ReadAllText(Path.Combine(rootPath, @"function3\function.json"))).Returns(function3); + fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"function3\function.json"), It.IsAny(), It.IsAny(), It.IsAny())).Returns(function3Stream); + return fileSystem.Object; }