Updates to GenericWorkerProvider (Azure#2975)

* Remove NodeWorkerProvider, Add support for parsing language Worker Arguments

schemas/json/host.json

@@ -1,8 +1,8 @@
 {
-    "title": "JSON schema for Azure Functions host.json files",
-    "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "JSON schema for Azure Functions host.json files",
+  "$schema": "http://json-schema.org/draft-04/schema#",
 
-    "type": "object",
+  "type": "object",
 
   "properties": {
     "id": {
@@ -254,6 +254,10 @@
           "type": "integer",
           "minimum": 4,
           "default": 32
+        },
+        "workersDirectory": {
+          "description": "Specifies full path of the directory for language workers",
+          "type": "string"
         }
       }
     },
@@ -301,4 +305,5 @@
         }
       }
     }
+  }
 }

src/WebJobs.Script.Grpc/Abstractions/IWorkerProvider.cs

@@ -24,7 +24,7 @@ public interface IWorkerProvider
         /// <param name="config">The host-level IConfiguration.</param>
         /// <param name="logger">The startup ILogger.</param>
         /// <returns>A bool that indicates if the args were configured successfully.</returns>
-        bool TryConfigureArguments(ArgumentsDescription args, IConfiguration config, ILogger logger);
+        bool TryConfigureArguments(WorkerProcessArguments args, IConfiguration config, ILogger logger);
 
         /// <summary>
         /// Get the worker directory path

src/WebJobs.Script.Grpc/Abstractions/WorkerDescription.cs

@@ -1,28 +1,53 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System.Collections.Generic;
+using System.IO;
+using Newtonsoft.Json;
+
 namespace Microsoft.Azure.WebJobs.Script.Abstractions
 {
     public class WorkerDescription
     {
         /// <summary>
         /// Gets or sets the name of the supported language. This is the same name as the IConfiguration section for the worker.
         /// </summary>
+        [JsonProperty(PropertyName = "language", Required = Required.Always)]
         public string Language { get; set; }
 
         /// <summary>
         /// Gets or sets the supported file extension type. Functions are registered with workers based on extension.
         /// </summary>
+        [JsonProperty(PropertyName = "extension", Required = Required.Always)]
         public string Extension { get; set; }
 
         /// <summary>
         /// Gets or sets the default executable path.
         /// </summary>
+        [JsonProperty(PropertyName = "defaultExecutablePath", Required = Required.Always)]
         public string DefaultExecutablePath { get; set; }
 
         /// <summary>
         /// Gets or sets the default path to the worker (relative to the bin/workers/{language} directory)
         /// </summary>
+        [JsonProperty(PropertyName = "defaultWorkerPath", Required = Required.Always)]
         public string DefaultWorkerPath { get; set; }
+
+        /// <summary>
+        /// Gets or sets the default base directory for the worker
+        /// </summary>
+        [JsonProperty(PropertyName = "workerDirectory")]
+        public string WorkerDirectory { get; set; }
+
+        /// <summary>
+        /// Gets or sets the default path to the worker (relative to the bin/workers/{language} directory)
+        /// </summary>
+        [JsonProperty(PropertyName = "arguments")]
+        public List<string> Arguments { get; set; }
+
+        public string GetWorkerPath()
+        {
+            return Path.Combine(WorkerDirectory, DefaultWorkerPath);
+        }
     }
 }

src/WebJobs.Script.Grpc/Abstractions/ArgumentsDescription.cs → src/WebJobs.Script.Grpc/Abstractions/WorkerProcessArguments.cs

@@ -5,7 +5,7 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Abstractions
 {
-    public class ArgumentsDescription
+    public class WorkerProcessArguments
     {
         /// <summary>
         /// Gets or sets the path to the executable (java, node, etc).

src/WebJobs.Script.Grpc/WebJobs.Script.Grpc.csproj

@@ -22,6 +22,7 @@
     </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Logging" Version="2.1.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.0-beta004">
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
@@ -31,6 +32,12 @@
     <Folder Include="Messages\DotNet\" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Reference Include="Newtonsoft.Json">
+      <HintPath>C:\Program Files\dotnet\sdk\NuGetFallbackFolder\newtonsoft.json\11.0.2\lib\netstandard2.0\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
   <Target Name="GenerateProtoFiles" BeforeTargets="BeforeBuild" Inputs="azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto" Outputs="Messages/DotNet/FunctionRpc.cs;Messages/DotNet/FunctionRpcGrpc.cs">
     <Exec Command="./generate_protos.$(Ext)" />
     <ItemGroup>

src/WebJobs.Script/Host/ScriptHost.cs

@@ -121,7 +121,7 @@ protected internal ScriptHost(IScriptHostEnvironment environment,
 
             _hostLogPath = Path.Combine(ScriptConfig.RootLogPath, "Host");
             _hostConfigFilePath = Path.Combine(ScriptConfig.RootScriptPath, ScriptConstants.HostMetadataFileName);
-            _language = _settingsManager.Configuration[ScriptConstants.FunctionWorkerRuntimeSettingName];
+            _language = _settingsManager.Configuration[LanguageWorkerConstants.FunctionWorkerRuntimeSettingName];
 
             _loggerProviderFactory = loggerProviderFactory ?? new DefaultLoggerProviderFactory();
         }
@@ -504,7 +504,7 @@ internal void InitializeFunctionDescriptors(Collection<FunctionMetadata> functio
                 _startupLogger.LogTrace($"Adding Function descriptor provider for language {_language}.");
                 switch (_language.ToLower())
                 {
-                    case ScriptConstants.DotNetLanguageWorkerName:
+                    case LanguageWorkerConstants.DotNetLanguageWorkerName:
                         _descriptorProviders.Add(new DotNetFunctionDescriptorProvider(this, ScriptConfig));
                         break;
                     default:
@@ -738,37 +738,33 @@ private void InitializeWorkers()
                     _hostConfig.LoggerFactory);
             };
 
+            var configFactory = new WorkerConfigFactory(ScriptSettingsManager.Instance.Configuration, _startupLogger);
             var providers = new List<IWorkerProvider>();
             if (!string.IsNullOrEmpty(_language))
             {
-                _startupLogger.LogInformation($"{ScriptConstants.FunctionWorkerRuntimeSettingName} is specified, only {_language} will be enabled");
+                _startupLogger.LogInformation($"'{LanguageWorkerConstants.FunctionWorkerRuntimeSettingName}' is specified, only '{_language}' will be enabled");
                 // TODO: We still have some hard coded languages, so we need to handle them. Remove this switch once we've moved away from that.
                 switch (_language.ToLowerInvariant())
                 {
-                    case ScriptConstants.NodeLanguageWorkerName:
-                        providers.Add(new NodeWorkerProvider());
-                        break;
-                    case ScriptConstants.JavaLanguageWrokerName:
-                        providers.Add(new JavaWorkerProvider());
+                    case LanguageWorkerConstants.JavaLanguageWorkerName:
+                        providers.Add(new JavaWorkerProvider(configFactory.WorkerDirPath));
                         break;
-                    case ScriptConstants.DotNetLanguageWorkerName:
+                    case LanguageWorkerConstants.DotNetLanguageWorkerName:
                         // No-Op
                         break;
                     default:
                         // Pass the language to the provider loader to filter
-                        providers.AddRange(GenericWorkerProvider.ReadWorkerProviderFromConfig(ScriptConfig, _startupLogger, language: _language));
+                        providers.AddRange(configFactory.GetWorkerProviders(_startupLogger, language: _language));
                         break;
                 }
             }
             else
             {
                 // load all providers if no specific language is specified
-                providers.Add(new NodeWorkerProvider());
-                providers.Add(new JavaWorkerProvider());
-                providers.AddRange(GenericWorkerProvider.ReadWorkerProviderFromConfig(ScriptConfig, _startupLogger));
+                providers.Add(new JavaWorkerProvider(configFactory.WorkerDirPath));
+                providers.AddRange(configFactory.GetWorkerProviders(_startupLogger));
             }
 
-            var configFactory = new WorkerConfigFactory(ScriptSettingsManager.Instance.Configuration, _startupLogger);
             var workerConfigs = configFactory.GetConfigs(providers);
 
             _functionDispatcher = new FunctionRegistry(EventManager, server, channelFactory, workerConfigs);
@@ -1650,19 +1646,19 @@ internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration
             }
             scriptConfig.HostConfig.FunctionTimeout = ScriptHost.CreateTimeoutConfiguration(scriptConfig);
 
-            ApplyLanguageWorkerConfig(config, scriptConfig, logger);
+            ApplyLanguageWorkersConfig(config, scriptConfig, logger);
             ApplyLoggerConfig(config, scriptConfig);
             ApplyApplicationInsightsConfig(config, scriptConfig);
         }
 
-        private static void ApplyLanguageWorkerConfig(JObject config, ScriptHostConfiguration scriptConfig, ILogger logger)
+        private static void ApplyLanguageWorkersConfig(JObject config, ScriptHostConfiguration scriptConfig, ILogger logger)
         {
             JToken value = null;
-            JObject languageWorkerSection = (JObject)config["languageWorker"];
+            JObject languageWorkersSection = (JObject)config[$"{LanguageWorkerConstants.LanguageWorkersSectionName}"];
             int requestedGrpcMaxMessageLength = ScriptSettingsManager.Instance.IsDynamicSku ? DefaultMaxMessageLengthBytesDynamicSku : DefaultMaxMessageLengthBytes;
-            if (languageWorkerSection != null)
+            if (languageWorkersSection != null)
             {
-                if (languageWorkerSection.TryGetValue("maxMessageLength", out value))
+                if (languageWorkersSection.TryGetValue("maxMessageLength", out value))
                 {
                     int valueInBytes = int.Parse((string)value) * 1024 * 1024;
                     if (ScriptSettingsManager.Instance.IsDynamicSku)

src/WebJobs.Script/Rpc/Configuration/GenericWorkerProvider.cs

@@ -1,8 +1,11 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
+
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
 using Microsoft.Azure.WebJobs.Script.Abstractions;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Extensions.Configuration;
@@ -14,13 +17,11 @@ namespace Microsoft.Azure.WebJobs.Script.Rpc
     internal class GenericWorkerProvider : IWorkerProvider
     {
         private WorkerDescription _workerDescription;
-        private List<string> _arguments;
         private string _pathToWorkerDir;
 
-        public GenericWorkerProvider(WorkerDescription workerDescription, List<string> arguments, string pathToWorkerDir)
+        public GenericWorkerProvider(WorkerDescription workerDescription, string pathToWorkerDir)
         {
             _workerDescription = workerDescription ?? throw new ArgumentNullException(nameof(workerDescription));
-            _arguments = arguments ?? throw new ArgumentNullException(nameof(arguments));
             _pathToWorkerDir = pathToWorkerDir ?? throw new ArgumentNullException(nameof(pathToWorkerDir));
         }
 
@@ -29,72 +30,18 @@ public WorkerDescription GetDescription()
             return _workerDescription;
         }
 
-        public bool TryConfigureArguments(ArgumentsDescription args, IConfiguration config, ILogger logger)
+        public bool TryConfigureArguments(WorkerProcessArguments args, IConfiguration config, ILogger logger)
         {
-            args.ExecutableArguments.AddRange(_arguments);
+            if (_workerDescription.Arguments != null)
+            {
+                args.ExecutableArguments.AddRange(_workerDescription.Arguments);
+            }
             return true;
         }
 
         public string GetWorkerDirectoryPath()
         {
             return _pathToWorkerDir;
         }
-
-        public static List<IWorkerProvider> ReadWorkerProviderFromConfig(ScriptHostConfiguration config, ILogger logger, ScriptSettingsManager settingsManager = null, string language = null)
-        {
-            var providers = new List<IWorkerProvider>();
-            settingsManager = settingsManager ?? ScriptSettingsManager.Instance;
-            var workerDirPath = settingsManager.Configuration.GetSection("workers:config:path").Value ?? WorkerProviderHelper.GetDefaultWorkerDirectoryPath();
-
-            if (!string.IsNullOrEmpty(language))
-            {
-                logger.LogInformation($"Reading Worker config for the language: {language}");
-                string languageWorkerDirectory = Path.Combine(workerDirPath, language);
-                var provider = GetProviderFromConfig(languageWorkerDirectory, logger);
-                if (provider != null)
-                {
-                    providers.Add(provider);
-                    logger.LogTrace($"Successfully added WorkerProvider for: {language}");
-                }
-            }
-            else
-            {
-                logger.LogInformation($"Loading all the worker providers from the default workers directory: {workerDirPath}");
-                foreach (var workerDir in Directory.EnumerateDirectories(workerDirPath))
-                {
-                    var provider = GetProviderFromConfig(workerDir, logger);
-                    if (provider != null)
-                    {
-                        providers.Add(provider);
-                    }
-                }
-            }
-            return providers;
-        }
-
-        public static IWorkerProvider GetProviderFromConfig(string workerDir, ILogger logger)
-        {
-            try
-            {
-                string workerConfigPath = Path.Combine(workerDir, ScriptConstants.WorkerConfigFileName);
-                if (File.Exists(workerConfigPath))
-                {
-                    logger.LogInformation($"Found worker config: {workerConfigPath}");
-                    string json = File.ReadAllText(workerConfigPath);
-                    JObject workerConfig = JObject.Parse(json);
-                    WorkerDescription workerDescription = workerConfig.Property(WorkerConstants.Description).Value.ToObject<WorkerDescription>();
-                    var arguments = new List<string>();
-                    arguments.AddRange(workerConfig.Property(WorkerConstants.Arguments).Value.ToObject<string[]>());
-                    logger.LogInformation($"Will load worker provider for language: {workerDescription.Language}");
-                    return new GenericWorkerProvider(workerDescription, arguments, workerDir);
-                }
-                return null;
-            }
-            catch (Exception ex)
-            {
-                logger?.LogError(ex, $"Failed to initialize worker provider for: {workerDir}");
-                return null;
-            }
-        }
     }
 }

src/WebJobs.Script/Rpc/Configuration/JavaWorkerProvider.cs

@@ -10,19 +10,26 @@ namespace Microsoft.Azure.WebJobs.Script.Rpc
 {
     internal class JavaWorkerProvider : IWorkerProvider
     {
-        private string pathToWorkerDir = WorkerProviderHelper.BuildWorkerDirectoryPath(ScriptConstants.JavaLanguageWrokerName);
+        private readonly string _pathToWorkerDir;
+
+        public JavaWorkerProvider(string workerDir)
+        {
+           _pathToWorkerDir = Path.Combine(workerDir, LanguageWorkerConstants.JavaLanguageWorkerName);
+        }
 
         public WorkerDescription GetDescription() => new WorkerDescription
         {
-            Language = ScriptConstants.JavaLanguageWrokerName,
+            Language = LanguageWorkerConstants.JavaLanguageWorkerName,
             Extension = ".jar",
             DefaultWorkerPath = "azure-functions-java-worker.jar",
+            WorkerDirectory = _pathToWorkerDir
         };
 
-        public bool TryConfigureArguments(ArgumentsDescription args, IConfiguration config, ILogger logger)
+        public bool TryConfigureArguments(WorkerProcessArguments args, IConfiguration config, ILogger logger)
         {
             var options = new DefaultWorkerOptions();
-            config.GetSection("workers:java").Bind(options);
+            var javaWorkerSection = $"{LanguageWorkerConstants.LanguageWorkersSectionName}:{LanguageWorkerConstants.JavaLanguageWorkerName}";
+            config.GetSection(javaWorkerSection).Bind(options);
             var env = new JavaEnvironment();
             config.Bind(env);
             if (string.IsNullOrEmpty(env.JAVA_HOME))
@@ -56,7 +63,7 @@ public bool TryConfigureArguments(ArgumentsDescription args, IConfiguration conf
 
         public string GetWorkerDirectoryPath()
         {
-            return pathToWorkerDir;
+            return _pathToWorkerDir;
         }
 
         private class JavaEnvironment