Hot Reload Validation (#2405)

## Why make this change?

This change fixes issue #2396 

## What is this change?

In order to validate the hot-reload, first, new logic was added to the
`RuntimeConfigLoader` and `FileSystemRuntimeConfigLoader`, so when hot
reload occurs, it updates the new states added to the loader which are
isNewConfigDetected, isNewConfigValidated, and lastValidRuntimeConfig.
Once the states are updated, it will try to parse and validate the new
RuntimeConfig, in the case that it fails at any step of the way, the
hot-reloading process will be canceled and the RuntimeConfig file will
be changed so that it uses the information from lastValidRuntimeConfig
so that the user can still use their last configuration without any
problems.

## How was this tested?

- [X] Integration Tests
- [ ] Unit Tests

---------

Co-authored-by: aaron burtle <[email protected]>
Co-authored-by: Ruben Cerna <[email protected]>
Co-authored-by: Aniruddh Munde <[email protected]>

src/Config/ConfigFileWatcher.cs

@@ -95,6 +95,15 @@ private void OnConfigFileChange(object sender, FileSystemEventArgs e)
                 }
             }
         }
+        catch (AggregateException ex)
+        {
+            // Need to remove the dependencies in startup on the RuntimeConfigProvider
+            // before we can have an ILogger here.
+            foreach (Exception exception in ex.InnerExceptions)
+            {
+                Console.WriteLine("Unable to hot reload configuration file due to " + exception.Message);
+            }
+        }
         catch (Exception ex)
         {
             // Need to remove the dependencies in startup on the RuntimeConfigProvider

src/Config/FileSystemRuntimeConfigLoader.cs

@@ -202,9 +202,20 @@ public bool TryLoadConfig(
                 }
 
                 config = RuntimeConfig;
+
+                if (LastValidRuntimeConfig is null)
+                {
+                    LastValidRuntimeConfig = RuntimeConfig;
+                }
+
                 return true;
             }
 
+            if (LastValidRuntimeConfig is not null)
+            {
+                RuntimeConfig = LastValidRuntimeConfig;
+            }
+
             config = null;
             return false;
         }
@@ -242,7 +253,16 @@ public override bool TryLoadKnownConfig([NotNullWhen(true)] out RuntimeConfig? c
     private void HotReloadConfig(ILogger? logger = null)
     {
         logger?.LogInformation(message: "Starting hot-reload process for config: {ConfigFilePath}", ConfigFilePath);
-        TryLoadConfig(ConfigFilePath, out _, replaceEnvVar: true);
+        if (!TryLoadConfig(ConfigFilePath, out _, replaceEnvVar: true))
+        {
+            throw new DataApiBuilderException(
+                message: "Deserialization of the configuration file failed.",
+                statusCode: HttpStatusCode.ServiceUnavailable,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
+        }
+
+        IsNewConfigDetected = true;
+        IsNewConfigValidated = false;
         SignalConfigChanged();
     }
 

src/Config/RuntimeConfigLoader.cs

@@ -32,6 +32,12 @@ public abstract class RuntimeConfigLoader
     // state in place of using out params.
     public RuntimeConfig? RuntimeConfig;
 
+    public RuntimeConfig? LastValidRuntimeConfig;
+
+    public bool IsNewConfigDetected;
+
+    public bool IsNewConfigValidated;
+
     public RuntimeConfigLoader(HotReloadEventHandler<HotReloadEventArgs>? handler = null, string? connectionString = null)
     {
         _changeToken = new DabChangeToken();
@@ -80,14 +86,14 @@ protected virtual void OnConfigChangedEvent(HotReloadEventArgs args)
     /// <param name="message"></param>
     protected void SignalConfigChanged(string message = "")
     {
+        // Signal that a change has occurred to all change token listeners.
+        RaiseChanged();
+
         OnConfigChangedEvent(new HotReloadEventArgs(QUERY_MANAGER_FACTORY_ON_CONFIG_CHANGED, message));
         OnConfigChangedEvent(new HotReloadEventArgs(METADATA_PROVIDER_FACTORY_ON_CONFIG_CHANGED, message));
         OnConfigChangedEvent(new HotReloadEventArgs(QUERY_ENGINE_FACTORY_ON_CONFIG_CHANGED, message));
         OnConfigChangedEvent(new HotReloadEventArgs(MUTATION_ENGINE_FACTORY_ON_CONFIG_CHANGED, message));
         OnConfigChangedEvent(new HotReloadEventArgs(DOCUMENTOR_ON_CONFIG_CHANGED, message));
-
-        // Signal that a change has occurred to all change token listeners.
-        RaiseChanged();
     }
 
     /// <summary>
@@ -339,4 +345,36 @@ internal static string GetPgSqlConnectionStringWithApplicationName(string connec
         // Return the updated connection string.
         return connectionStringBuilder.ConnectionString;
     }
+
+    public bool DoesConfigNeedValidation()
+    {
+        if (IsNewConfigDetected && !IsNewConfigValidated)
+        {
+            IsNewConfigDetected = false;
+            return true;
+        }
+
+        return false;
+    }
+
+    /// <summary>
+    /// Once the validation of the new config file is confirmed to have passed,
+    /// this function will save the newly resolved RuntimeConfig as the new last known good,
+    /// in order to have config file DAB can go into in case hot reload fails.
+    /// </summary>
+    public void SetLkgConfig()
+    {
+        IsNewConfigValidated = false;
+        LastValidRuntimeConfig = RuntimeConfig;
+    }
+
+    /// <summary>
+    /// Changes the state of the config file into the last known good iteration,
+    /// in order to allow users to still be able to make changes in DAB even if
+    /// a hot reload fails.
+    /// </summary>
+    public void RestoreLkgConfig()
+    {
+        RuntimeConfig = LastValidRuntimeConfig;
+    }
 }

src/Core/Configurations/RuntimeConfigProvider.cs

@@ -3,12 +3,14 @@
 
 using System.Data.Common;
 using System.Diagnostics.CodeAnalysis;
+using System.IO.Abstractions;
 using System.Net;
 using Azure.DataApiBuilder.Config;
 using Azure.DataApiBuilder.Config.Converters;
 using Azure.DataApiBuilder.Config.NamingPolicies;
 using Azure.DataApiBuilder.Config.ObjectModel;
 using Azure.DataApiBuilder.Service.Exceptions;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 
 namespace Azure.DataApiBuilder.Core.Configurations;
@@ -63,6 +65,11 @@ public RuntimeConfigProvider(RuntimeConfigLoader runtimeConfigLoader)
     /// </seealso>
     private void RaiseChanged()
     {
+        //First use of GetConfig during hot reload, in order to do validation of
+        //config file before any changes are made for hot reload.
+        //In case validation fails, an exception will be thrown and hot reload will be canceled.
+        ValidateConfig();
+
         DabChangeToken previousToken = Interlocked.Exchange(ref _changeToken, new DabChangeToken());
         previousToken.SignalChange();
     }
@@ -290,6 +297,41 @@ public bool IsConfigHotReloadable()
         return !IsLateConfigured || !(_configLoader.RuntimeConfig?.Runtime?.Host?.Mode == HostMode.Production);
     }
 
+    /// <summary>
+    /// This function checks if there is a new config that needs to be validated
+    /// and validates the configuration file as well as the schema file, in the
+    /// case that it is not able to validate both then it will return an error.
+    /// </summary>
+    /// <returns></returns>
+    public void ValidateConfig()
+    {
+        // Only used in hot reload to validate the configuration file
+        if (_configLoader.DoesConfigNeedValidation())
+        {
+            IFileSystem fileSystem = new FileSystem();
+            ILoggerFactory loggerFactory = new LoggerFactory();
+            ILogger<RuntimeConfigValidator> logger = loggerFactory.CreateLogger<RuntimeConfigValidator>();
+            RuntimeConfigValidator runtimeConfigValidator = new(this, fileSystem, logger, true);
+
+            _configLoader.IsNewConfigValidated = runtimeConfigValidator.TryValidateConfig(ConfigFilePath, loggerFactory).Result;
+
+            // Saves the lastValidRuntimeConfig as the new RuntimeConfig if it is validated for hot reload
+            if (_configLoader.IsNewConfigValidated)
+            {
+                _configLoader.SetLkgConfig();
+            }
+            else
+            {
+                _configLoader.RestoreLkgConfig();
+
+                throw new DataApiBuilderException(
+                    message: "Failed validation of configuration file.",
+                    statusCode: HttpStatusCode.ServiceUnavailable,
+                    subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
+            }
+        }
+    }
+
     private async Task<bool> InvokeConfigLoadedHandlersAsync()
     {
         List<Task<bool>> configLoadedTasks = new();

src/Service.Tests/Configuration/ConfigurationTests.cs

@@ -4144,6 +4144,125 @@ public async Task TestNoDepthLimitOnGrahQLInNonHostedMode(int? depthLimit)
             }
         }
 
+        /// <summary>
+        /// Creates a hot reload scenario in which the schema file is invalid which causes
+        /// hot reload to fail, then we check that the program is still able to work
+        /// properly by validating that the DAB engine is still using the same configuration file
+        /// from before the hot reload.
+        /// </summary>
+        [TestMethod]
+        [TestCategory(TestCategory.MSSQL)]
+        public void HotReloadValidationFail()
+        {
+            // Arrange
+            string schemaName = "testSchema.json";
+            string configName = "hotreload-config.json";
+            if (File.Exists(configName))
+            {
+                File.Delete(configName);
+            }
+
+            if (File.Exists(schemaName))
+            {
+                File.Delete(schemaName);
+            }
+
+            bool initialRestEnabled = true;
+            bool updatedRestEnabled = false;
+
+            bool initialGQLEnabled = true;
+            bool updatedGQLEnabled = false;
+
+            DataSource dataSource = new(DatabaseType.MSSQL,
+                ConfigurationTests.GetConnectionStringFromEnvironmentConfig(environment: TestCategory.MSSQL), Options: null);
+
+            RuntimeConfig initialConfig = InitRuntimeConfigForHotReload(schemaName, dataSource, initialRestEnabled, initialGQLEnabled);
+
+            RuntimeConfig updatedConfig = InitRuntimeConfigForHotReload(schemaName, dataSource, updatedRestEnabled, updatedGQLEnabled);
+
+            string schemaConfig = TestHelper.GenerateInvalidSchema();
+
+            // Not using mocked filesystem so we pick up real file changes for hot reload
+            FileSystem fileSystem = new();
+            RuntimeConfigProvider configProvider = GenerateConfigFileAndConfigProvider(fileSystem, configName, initialConfig);
+            RuntimeConfig lkgRuntimeConfig = configProvider.GetConfig();
+
+            Assert.IsNotNull(lkgRuntimeConfig);
+
+            // Act
+            // Simulate an invalid change to the schema file while the config is updated to a valid state
+            fileSystem.File.WriteAllText(schemaName, schemaConfig);
+            fileSystem.File.WriteAllText(configName, updatedConfig.ToJson());
+
+            // Give ConfigFileWatcher enough time to hot reload the change
+            System.Threading.Thread.Sleep(6000);
+
+            RuntimeConfig newRuntimeConfig = configProvider.GetConfig();
+            Assert.AreEqual(expected: lkgRuntimeConfig, actual: newRuntimeConfig);
+
+            if (File.Exists(configName))
+            {
+                File.Delete(configName);
+            }
+
+            if (File.Exists(schemaName))
+            {
+                File.Delete(schemaName);
+            }
+        }
+
+        /// <summary>
+        /// Creates a hot reload scenario in which the updated configuration file is invalid causing
+        /// hot reload to fail as the schema can't be used by DAB, then we check that the
+        /// program is still able to work properly by showing us that it is still using the
+        /// same configuration file from before the hot reload.
+        /// </summary>
+        [TestMethod]
+        [TestCategory(TestCategory.MSSQL)]
+        public void HotReloadParsingFail()
+        {
+            // Arrange
+            string schemaName = "dab.draft.schema.json";
+            string configName = "hotreload-config.json";
+            if (File.Exists(configName))
+            {
+                File.Delete(configName);
+            }
+
+            bool initialRestEnabled = true;
+
+            bool initialGQLEnabled = true;
+
+            DataSource dataSource = new(DatabaseType.MSSQL,
+                ConfigurationTests.GetConnectionStringFromEnvironmentConfig(environment: TestCategory.MSSQL), Options: null);
+
+            RuntimeConfig initialConfig = InitRuntimeConfigForHotReload(schemaName, dataSource, initialRestEnabled, initialGQLEnabled);
+
+            string updatedConfig = TestHelper.GenerateInvalidRuntimeSection();
+
+            // Not using mocked filesystem so we pick up real file changes for hot reload
+            FileSystem fileSystem = new();
+            RuntimeConfigProvider configProvider = GenerateConfigFileAndConfigProvider(fileSystem, configName, initialConfig);
+            RuntimeConfig lkgRuntimeConfig = configProvider.GetConfig();
+
+            Assert.IsNotNull(lkgRuntimeConfig);
+
+            // Act
+            // Simulate an invalid change to the config file
+            fileSystem.File.WriteAllText(configName, updatedConfig);
+
+            // Give ConfigFileWatcher enough time to hot reload the change
+            System.Threading.Thread.Sleep(1000);
+
+            RuntimeConfig newRuntimeConfig = configProvider.GetConfig();
+            Assert.AreEqual(expected: lkgRuntimeConfig, actual: newRuntimeConfig);
+
+            if (File.Exists(configName))
+            {
+                File.Delete(configName);
+            }
+        }
+
         /// <summary>
         /// Helper function to write custom configuration file. with minimal REST/GraphQL global settings
         /// using the supplied entities.
@@ -4596,6 +4715,53 @@ public static RuntimeConfig InitMinimalRuntimeConfig(
             );
         }
 
+        /// <summary>
+        /// Helper function that initializes a RuntimeConfig with the following
+        /// variables in order to prepare a file for hot reload
+        /// </summary>
+        /// <param name="schema"></param>
+        /// <param name="dataSource"></param>
+        /// <param name="restEnabled"></param>
+        /// <param name="graphQLEnabled"></param>
+        /// <returns></returns>
+        public static RuntimeConfig InitRuntimeConfigForHotReload(
+            string schema,
+            DataSource dataSource,
+            bool restEnabled,
+            bool graphQLEnabled)
+        {
+            RuntimeConfig runtimeConfig = new(
+                Schema: schema,
+                DataSource: dataSource,
+                Runtime: new(
+                    Rest: new(restEnabled),
+                    GraphQL: new(graphQLEnabled),
+                    Host: new(null, null, HostMode.Development)
+                ),
+                Entities: new(new Dictionary<string, Entity>())
+            );
+
+            return runtimeConfig;
+        }
+
+        /// <summary>
+        /// Helper function that generates a config file that is going to be observed
+        /// for hot reload and initialize a ConfigProvider which will be used to check
+        /// if the hot reload was validated or not.
+        /// </summary>
+        /// <param name="fileSystem"></param>
+        /// <param name="configName"></param>
+        /// <param name="runtimeConfig"></param>
+        /// <returns></returns>
+        public static RuntimeConfigProvider GenerateConfigFileAndConfigProvider(FileSystem fileSystem, string configName, RuntimeConfig runtimeConfig)
+        {
+            fileSystem.File.WriteAllText(configName, runtimeConfig.ToJson());
+            FileSystemRuntimeConfigLoader configLoader = new(fileSystem, handler: null, configName, string.Empty);
+            RuntimeConfigProvider configProvider = new(configLoader);
+
+            return configProvider;
+        }
+
         /// <summary>
         /// Gets PermissionSetting object allowed to perform all actions.
         /// </summary>