Fix Overflow bug in RandomDataGenerator (QuantConnect#7423)

* First attempt to fix the bug

* Allow the splitFactor to change over time

- Add unit test

* Nit change

* Address required changes

* Address required changes

* Adjust upper bound

Tests/ToolBox/RandomDataGenerator/DividendSplitMapGeneratorTests.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using NUnit.Framework;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class DividendSplitMapGeneratorTests
+    {
+        [TestCase(240, 0.9857119009006162)]
+        [TestCase(120, 0.9716279515771061)]
+        [TestCase(60, 0.9440608762859234)]
+        [TestCase(12, 0.7498942093324559)]
+        [TestCase(6, 0.5623413251903491)]
+        [TestCase(3, 0.31622776601683794)]
+        [TestCase(1, 0.03162277660168379)]
+        public void GetsExpectedLowerBound(int months, double expectedLowerBound)
+        {
+            var lowerBound = (double)DividendSplitMapGenerator.GetLowerBoundForPreviousSplitFactor(months);
+            Assert.AreEqual(expectedLowerBound, lowerBound, delta: 0.0000000000001);
+            Assert.IsTrue(Math.Pow(lowerBound, lowerBound * 2) >= 0.0009);
+        }
+
+        [TestCase(240)]
+        [TestCase(120)]
+        [TestCase(60)]
+        [TestCase(12)]
+        [TestCase(6)]
+        [TestCase(3)]
+        [TestCase(1)]
+        public void GetsValidNextPreviousSplitFactor(int months)
+        {
+            var lowerBound = DividendSplitMapGenerator.GetLowerBoundForPreviousSplitFactor(months);
+            var upperBound = 1;
+            var nextPreviousSplitFactor = DividendSplitMapGenerator.GetNextPreviousSplitFactor(new Random(), lowerBound, upperBound);
+            Assert.IsTrue(lowerBound <= nextPreviousSplitFactor && nextPreviousSplitFactor <= upperBound);
+            Assert.IsTrue(0.001 <= Math.Pow((double)nextPreviousSplitFactor, 2 * (double)months) && Math.Pow((double)nextPreviousSplitFactor, 2 * (double)months) <= 1);
+        }
+
+        [TestCase(240)]
+        [TestCase(120)]
+        [TestCase(60)]
+        [TestCase(12)]
+        [TestCase(6)]
+        [TestCase(3)]
+        [TestCase(1)]
+        public void PriceScaledBySplitFactorIsBounded(int months)
+        {
+            var maxPossiblePrice = 1000000m;
+            var minPossiblePrice = 0.0001m;
+            var lowerBound = DividendSplitMapGenerator.GetLowerBoundForPreviousSplitFactor(months);
+            var upperBound = 1;
+            var nextPreviousSplitFactor = DividendSplitMapGenerator.GetNextPreviousSplitFactor(new Random(), lowerBound, upperBound);
+            var finalSplitFactor = (decimal)Math.Pow((double)nextPreviousSplitFactor, 2 * (double)months);
+            Assert.IsTrue(0.0001m <= (maxPossiblePrice / finalSplitFactor) && (maxPossiblePrice / finalSplitFactor) <= 1000000000m, (maxPossiblePrice / finalSplitFactor).ToString());
+            Assert.IsTrue(0.0001m <= (minPossiblePrice / finalSplitFactor) && (minPossiblePrice / finalSplitFactor) <= 1000000000m, (minPossiblePrice / finalSplitFactor).ToString());
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs

@@ -14,8 +14,19 @@
 */
 
 using System;
+using System.Collections.Generic;
+using QuantConnect.Securities;
 using NUnit.Framework;
+using QuantConnect.Data;
 using QuantConnect.ToolBox.RandomDataGenerator;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities.Option;
+using QuantConnect.Util;
+using static QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator;
 
 namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
 {
@@ -44,5 +55,97 @@ public void NextRandomGeneratedData(DateTime start, DateTime end, DateTime expec
             Assert.LessOrEqual(delistDate, end);
             Assert.GreaterOrEqual(delistDate, midPoint);
         }
+
+        [TestCase("20220101", "20230101")]
+        public void RandomGeneratorProducesValuesBoundedForEquitiesWhenSplit(string start, string end)
+        {
+            var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
+                start,
+                end,
+                "1",
+                "usa",
+                "Equity",
+                "Minute",
+                "Dense",
+                "true",
+                "1",
+                null,
+                "5.0",
+                "30.0",
+                "100.0",
+                "60.0",
+                "30.0",
+                "BaroneAdesiWhaleyApproximationEngine",
+                "Daily",
+                "1",
+                new List<string>(),
+                100
+            );
+
+            var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));
+            var securityService = GetSecurityService(settings, securityManager);
+            securityManager.SetSecurityService(securityService);
+
+            var security = securityManager.CreateSecurity(Symbols.AAPL, new List<SubscriptionDataConfig>(), underlying: null);
+            var randomValueGenerator = new RandomValueGenerator();
+            var tickGenerator = new TickGenerator(settings, new TickType[1] {TickType.Trade}, security, randomValueGenerator).GenerateTicks().GetEnumerator();
+            using var sync = new SynchronizingBaseDataEnumerator(tickGenerator);
+            var tickHistory = new List<Tick>();
+
+            while (sync.MoveNext())
+            {
+                var dataPoint = sync.Current;
+                tickHistory.Add(dataPoint as Tick);
+            }
+
+            var dividendsSplitsMaps = new DividendSplitMapGenerator(
+                        Symbols.AAPL,
+                        settings,
+                        randomValueGenerator,
+                        BaseSymbolGenerator.Create(settings, randomValueGenerator),
+                        new Random(),
+                        GetDelistingDate(settings.Start, settings.End, randomValueGenerator),
+                        false);
+
+            dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
+            Assert.IsTrue(0.099m <= dividendsSplitsMaps.FinalSplitFactor && dividendsSplitsMaps.FinalSplitFactor <= 1.5m);
+
+            foreach (var tick in tickHistory)
+            {
+                tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
+                Assert.IsTrue( 0.001m <= tick.Value && tick.Value <= 1000000000, $"The tick value was {tick.Value} but should have been bounded by 0.001 and 1 000 000 000");
+            }
+        }
+
+        private static SecurityService GetSecurityService(RandomDataGeneratorSettings settings, SecurityManager securityManager)
+        {
+            var securityService = new SecurityService(
+                new CashBook(),
+                MarketHoursDatabase.FromDataFolder(),
+                SymbolPropertiesDatabase.FromDataFolder(),
+                new SecurityInitializerProvider(new FuncSecurityInitializer(security =>
+                {
+                    // init price
+                    security.SetMarketPrice(new Tick(settings.Start, security.Symbol, 100, 100));
+                    security.SetMarketPrice(new OpenInterest(settings.Start, security.Symbol, 10000));
+
+                    // from settings
+                    security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(settings.VolatilityModelResolution);
+
+                    // from settings
+                    if (security is Option option)
+                    {
+                        option.PriceModel = OptionPriceModels.Create(settings.OptionPriceEngineName, Statistics.PortfolioStatistics.GetRiskFreeRate());
+                    }
+                })),
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCacheProvider(
+                    new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager), new AlgorithmSettings())),
+                new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider")))
+            );
+            securityManager.SetSecurityService(securityService);
+
+            return securityService;
+        }
     }
 }

ToolBox/RandomDataGenerator/DividendSplitMapGenerator.cs

@@ -1,8 +1,21 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.Auxiliary;
 
@@ -13,6 +26,8 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
     /// </summary>
     public class DividendSplitMapGenerator
     {
+        private const double _minimumFinalSplitFactorAllowed = 0.001;
+
         /// <summary>
         /// The final factor to adjust all prices with in order to maintain price continuity.
         /// </summary>
@@ -77,7 +92,20 @@ public void GenerateSplitsDividends(IEnumerable<Tick> tickHistory)
             var dividendEveryQuarter = _randomValueGenerator.NextBool(_settings.DividendEveryQuarterPercentage);
 
             var previousX = _random.NextDouble();
-            var previousSplitFactor = hasSplits ? (decimal)_random.NextDouble() : 1;
+
+            // Since the largest equity value we can obtain is 1 000 000, if we want this price divided by the FinalSplitFactor
+            // to be upper bounded by 1 000 000 000 we need to make sure the FinalSplitFactor is lower bounded by 0.001. Therefore,
+            // since in the worst of the cases FinalSplitFactor = (previousSplitFactor)^(2m), where m is the number of months
+            // in the time span, we need to lower bound previousSplitFactor by (0.001)^(1/(2m))
+            //
+            // On the other hand, if the upper bound for the previousSplitFactor is 1, then the FinalSplitFactor will be, in the
+            // worst of the cases as small as the minimum equity value we can obtain
+
+            var months = (int)Math.Round(_settings.End.Subtract(_settings.Start).Days / (365.25 / 12));
+            months = months != 0 ? months : 1;
+            var minPreviousSplitFactor = GetLowerBoundForPreviousSplitFactor(months);
+            var maxPreviousSplitFactor = 1;
+            var previousSplitFactor = hasSplits ? GetNextPreviousSplitFactor(_random, minPreviousSplitFactor, maxPreviousSplitFactor) : 1;
             var previousPriceFactor = hasDividends ? (decimal)Math.Tanh(previousX) : 1;
 
             var splitDates = new List<DateTime>();
@@ -165,19 +193,21 @@ public void GenerateSplitsDividends(IEnumerable<Tick> tickHistory)
                             }
                         }
                         // Have a 5% chance of a split every month
-                        if (hasSplits && _randomValueGenerator.NextBool(5.0))
+                        if (hasSplits && _randomValueGenerator.NextBool(_settings.MonthSplitPercentage))
                         {
-                            do
+                            // Produce another split factor that is also bounded by the min and max split factors allowed
+                            if (_randomValueGenerator.NextBool(5.0)) // Add the possibility of a reverse split
                             {
-                                if (previousSplitFactor < 1)
-                                {
-                                    previousSplitFactor += (decimal)(_random.NextDouble() - _random.NextDouble());
-                                }
-                                else
-                                {
-                                    previousSplitFactor *= (decimal)_random.NextDouble() * _random.Next(1, 5);
-                                }
-                            } while (previousSplitFactor < 0);
+                                // A reverse split is a split that is smaller than the current previousSplitFactor
+                                // Update previousSplitFactor with a smaller value that is still bounded below by minPreviousSplitFactor
+                                previousSplitFactor = GetNextPreviousSplitFactor(_random, minPreviousSplitFactor, previousSplitFactor);
+                            }
+                            else
+                            {
+                                // Update previousSplitFactor with a higher value that is still bounded by maxPreviousSplitFactor
+                                // Usually, the split factor tends to grow across the time span(See /Data/Equity/usa/factor_files/aapl for instance)
+                                previousSplitFactor = GetNextPreviousSplitFactor(_random, previousSplitFactor, maxPreviousSplitFactor);
+                            }
 
                             splitDates.Add(_randomValueGenerator.NextDate(tick.Time, tick.Time.AddMonths(1), (DayOfWeek)_random.Next(1, 5)));
                         }
@@ -204,5 +234,32 @@ public void GenerateSplitsDividends(IEnumerable<Tick> tickHistory)
                 }
             }
         }
+
+        /// <summary>
+        /// Gets a lower bound that guarantees the FinalSplitFactor, in all the possible
+        /// cases, will never be smaller than the _minimumFinalSplitFactorAllowed (0.001)
+        /// </summary>
+        /// <param name="months">The lower bound for the previous split factor is based on
+        /// the number of months between the start and end date from ticksHistory <see cref="GenerateSplitsDividends(IEnumerable{Tick})"></param>
+        /// <returns>A valid lower bound that guarantees the FinalSplitFactor is always higher
+        /// than the _minimumFinalSplitFactorAllowed</returns>
+        public static decimal GetLowerBoundForPreviousSplitFactor(int months)
+        {
+            return (decimal)(Math.Pow(_minimumFinalSplitFactorAllowed, 1 / (double)(2 * months)));
+        }
+
+        /// <summary>
+        /// Gets a new valid previousSplitFactor that is still bounded by the given upper and lower
+        /// bounds
+        /// </summary>
+        /// <param name="random">Random number generator</param>
+        /// <param name="lowerBound">Minimum allowed value to obtain</param>
+        /// <param name="upperBound">Maximum allowed value to obtain</param>
+        /// <returns>A new valid previousSplitFactor that is still bounded by the given upper and lower
+        /// bounds</returns>
+        public static decimal GetNextPreviousSplitFactor(Random random, decimal lowerBound, decimal upperBound)
+        {
+            return ((decimal)random.NextDouble()) * (upperBound - lowerBound) + lowerBound;
+        }
     }
 }

ToolBox/RandomDataGenerator/RandomDataGeneratorSettings.cs

@@ -43,6 +43,7 @@ public class RandomDataGeneratorSettings
         public double HasIpoPercentage { get; init; }
         public double HasRenamePercentage { get; init; }
         public double HasSplitsPercentage { get; init; }
+        public double MonthSplitPercentage { get; init; }
         public double HasDividendsPercentage { get; init; }
         public double DividendEveryQuarterPercentage { get; init; }
         public string OptionPriceEngineName { get; init; }
@@ -68,7 +69,8 @@ public static RandomDataGeneratorSettings FromCommandLineArguments(
             string optionPriceEngineName,
             string volatilityModelResolutionString,
             string chainSymbolCountString,
-            List<string> tickers
+            List<string> tickers,
+            double monthSplitPercentage = 5.0
             )
         {
             var randomSeedSet = true;
@@ -319,6 +321,7 @@ List<string> tickers
                 HasIpoPercentage = hasIpoPercentage,
                 HasRenamePercentage = hasRenamePercentage,
                 HasSplitsPercentage = hasSplitsPercentage,
+                MonthSplitPercentage = monthSplitPercentage,
                 HasDividendsPercentage = hasDividendsPercentage,
                 DividendEveryQuarterPercentage = dividendEveryQuarterPercentage,
                 OptionPriceEngineName = optionPriceEngineName,

ToolBox/RandomDataGenerator/RandomValueGenerator.cs

@@ -16,7 +16,6 @@
 using QuantConnect.Securities;
 using QuantConnect.Util;
 using System;
-using System.Collections.Generic;
 using System.Linq;
 
 namespace QuantConnect.ToolBox.RandomDataGenerator
@@ -30,6 +29,7 @@ public class RandomValueGenerator : IRandomValueGenerator
         private readonly Random _random;
         private readonly MarketHoursDatabase _marketHoursDatabase;
         private readonly SymbolPropertiesDatabase _symbolPropertiesDatabase;
+        private const decimal _maximumPriceAllowed = 1000000m;
 
 
         public RandomValueGenerator()
@@ -173,10 +173,24 @@ public virtual decimal NextPrice(SecurityType securityType, string market, decim
                 if (price < 20 * minimumPriceVariation)
                 {
                     // The price should not be to close to the minimum price variation.
-                    // Invalidate the price to try again and increase the probability of the it going up
+                    // Invalidate the price to try again and increase the probability of it to going up
                     price = -1m;
                     increaseProbabilityFactor = Math.Max(increaseProbabilityFactor - 0.05, 0);
                 }
+
+                if (price > (_maximumPriceAllowed / 10m))
+                {
+                    // The price should not be too higher
+                    // Decrease the probability of it to going up
+                    increaseProbabilityFactor = increaseProbabilityFactor + 0.05;
+                }
+
+                if (price > _maximumPriceAllowed)
+                {
+                    // The price should not be too higher
+                    // Invalidate the price to try again
+                    price = -1;
+                }
             } while (!IsPriceValid(securityType, price) && ++attempts < 10);
 
             if (!IsPriceValid(securityType, price))
@@ -209,7 +223,7 @@ private static bool IsPriceValid(SecurityType securityType, decimal price)
                 }
                 default:
                 {
-                    return price > 0;
+                    return price > 0 && price < _maximumPriceAllowed;
                 }
             }
         }