Normalize ETF constituents universe (QuantConnect#7858)

- Normalize ETF constituents universe data. Adding new tests and
  assertions

Algorithm.CSharp/ETFConstituentsFrameworkAlgorithm.cs

@@ -44,7 +44,7 @@ public override void Initialize()
             SetPortfolioConstruction(new EqualWeightingPortfolioConstructionModel());
         }
 
-        private protected IEnumerable<Symbol> ETFConstituentsFilter(IEnumerable<ETFConstituentData> constituents)
+        private protected IEnumerable<Symbol> ETFConstituentsFilter(IEnumerable<ETFConstituentUniverse> constituents)
         {
             // Get the 10 securities with the largest weight in the index
             return constituents.OrderByDescending(c => c.Weight).Take(8).Select(c => c.Symbol);

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.cs

@@ -66,7 +66,7 @@ public override void Initialize()
             AddUniverse(Universe.ETF(_gdvd, universeFilterFunc: FilterETFs));
         }
 
-        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentData> constituents)
+        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentUniverse> constituents)
         {
             _universeSelectionDone = true;
 

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseFilterFunctionRegressionAlgorithm.cs

@@ -27,7 +27,7 @@ namespace QuantConnect.Algorithm.CSharp
     /// </summary>
     public class ETFConstituentUniverseFilterFunctionRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        private Dictionary<Symbol, ETFConstituentData> _etfConstituentData = new Dictionary<Symbol, ETFConstituentData>();
+        private Dictionary<Symbol, ETFConstituentUniverse> _etfConstituentData = new Dictionary<Symbol, ETFConstituentUniverse>();
 
         private Symbol _aapl;
         private Symbol _spy;
@@ -61,7 +61,7 @@ public override void Initialize()
         /// <param name="constituents">Constituents of the ETF universe added above</param>
         /// <returns>Constituent Symbols to add to algorithm</returns>
         /// <exception cref="ArgumentException">Constituents collection was not structured as expected</exception>
-        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentData> constituents)
+        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentUniverse> constituents)
         {
             var constituentsData = constituents.ToList();
             _etfConstituentData = constituentsData.ToDictionary(x => x.Symbol, x => x);

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseFrameworkRegressionAlgorithm.cs

@@ -30,7 +30,7 @@ namespace QuantConnect.Algorithm.CSharp
     /// </summary>
     public class ETFConstituentUniverseFrameworkRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        private List<ETFConstituentData> ConstituentData = new List<ETFConstituentData>();
+        private List<ETFConstituentUniverse> ConstituentData = new List<ETFConstituentUniverse>();
 
         /// <summary>
         /// Initializes the algorithm, setting up the framework classes and ETF constituent universe settings
@@ -53,15 +53,28 @@ public override void Initialize()
 
         protected virtual void AddUniverseWrapper(Symbol symbol)
         {
-            AddUniverse(Universe.ETF(symbol, UniverseSettings, FilterETFConstituents));
+            var universe = AddUniverse(Universe.ETF(symbol, UniverseSettings, FilterETFConstituents));
+
+            var historicalData = History(universe, 1).ToList();
+            if (historicalData.Count != 1)
+            {
+                throw new Exception($"Unexpected history count {historicalData.Count}! Expected 1");
+            }
+            foreach (var universeDataCollection in historicalData)
+            {
+                if (universeDataCollection.Data.Count < 200)
+                {
+                    throw new Exception($"Unexpected universe DataCollection count {universeDataCollection.Data.Count}! Expected > 200");
+                }
+            }
         }
 
         /// <summary>
         /// Filters ETF constituents
         /// </summary>
         /// <param name="constituents">ETF constituents</param>
         /// <returns>ETF constituent Symbols that we want to include in the algorithm</returns>
-        public IEnumerable<Symbol> FilterETFConstituents(IEnumerable<ETFConstituentData> constituents)
+        public IEnumerable<Symbol> FilterETFConstituents(IEnumerable<ETFConstituentUniverse> constituents)
         {
             var constituentData = constituents
                 .Where(x => (x.Weight ?? 0m) >= 0.001m)
@@ -210,7 +223,7 @@ public void Execute(QCAlgorithm algorithm, IPortfolioTarget[] targets)
         /// <summary>
         /// Data Points count of the algorithm history
         /// </summary>
-        public int AlgorithmHistoryDataPoints => 0;
+        public virtual int AlgorithmHistoryDataPoints => 1;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseFrameworkRegressionAlgorithmNewUniverseModel.cs

@@ -28,5 +28,7 @@ protected override void AddUniverseWrapper(Symbol symbol)
         }
 
         public override Language[] Languages { get; } = { Language.CSharp };
+
+        public override int AlgorithmHistoryDataPoints => 0;
     }
 }

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseImmediateSelectionRegressionAlgorithm.cs

@@ -56,7 +56,7 @@ public override void Initialize()
         /// <param name="constituents">Constituents of the ETF universe added above</param>
         /// <returns>Constituent Symbols to add to algorithm</returns>
         /// <exception cref="ArgumentException">Constituents collection was not structured as expected</exception>
-        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentData> constituents)
+        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentUniverse> constituents)
         {
             _filtered = true;
             _constituents = constituents.Select(x => x.Symbol).Distinct().ToList();

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseMappedCompositeRegressionAlgorithm.cs

@@ -51,7 +51,7 @@ public override void Initialize()
             AddUniverse(Universe.ETF(_qqq, universeFilterFunc: FilterETFs));
         }
 
-        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentData> constituents)
+        private IEnumerable<Symbol> FilterETFs(IEnumerable<ETFConstituentUniverse> constituents)
         {
             var constituentSymbols = constituents.Select(x => x.Symbol).ToHashSet();
             if (!constituentSymbols.Contains(_aapl))

Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseRSIAlphaModelAlgorithm.cs

@@ -60,7 +60,7 @@ public override void Initialize()
         /// </summary>
         /// <param name="constituents">ETF constituents, i.e. the components of the ETF and their weighting</param>
         /// <returns>Symbols to add to universe</returns>
-        public IEnumerable<Symbol> FilterETFConstituents(IEnumerable<ETFConstituentData> constituents)
+        public IEnumerable<Symbol> FilterETFConstituents(IEnumerable<ETFConstituentUniverse> constituents)
         {
             return constituents
                 .Where(x => x.Weight != null && x.Weight >= 0.001m)
@@ -85,8 +85,8 @@ public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)
             {
                 // Cast first, and then access the constituents collection defined in our algorithm.
                 var algoConstituents = data.Bars.Keys
-                    .Where(x => algorithm.Securities[x].Cache.HasData(typeof(ETFConstituentData)))
-                    .Select(x => algorithm.Securities[x].Cache.GetData<ETFConstituentData>())
+                    .Where(x => algorithm.Securities[x].Cache.HasData(typeof(ETFConstituentUniverse)))
+                    .Select(x => algorithm.Securities[x].Cache.GetData<ETFConstituentUniverse>())
                     .ToList();
 
                 if (algoConstituents.Count == 0 || data.Bars.Count == 0)
@@ -167,7 +167,7 @@ private class SymbolData
             /// <summary>
             /// Symbol's constituent data for the ETF it belongs to
             /// </summary>
-            public ETFConstituentData Constituent { get; }
+            public ETFConstituentUniverse Constituent { get; }
 
             /// <summary>
             /// RSI indicator for the Symbol's price data
@@ -180,7 +180,7 @@ private class SymbolData
             /// <param name="symbol">The symbol to add data for</param>
             /// <param name="constituent">ETF constituent data</param>
             /// <param name="period">RSI period</param>
-            public SymbolData(Symbol symbol, QCAlgorithm algorithm, ETFConstituentData constituent, int period)
+            public SymbolData(Symbol symbol, QCAlgorithm algorithm, ETFConstituentUniverse constituent, int period)
             {
                 Symbol = symbol;
                 Constituent = constituent;

Algorithm.CSharp/UniverseOnlyRegressionAlgorithm.cs

@@ -49,7 +49,7 @@ public override void OnEndOfAlgorithm()
             }
         }
 
-        private IEnumerable<Symbol> FilterUniverse(IEnumerable<ETFConstituentData> constituents)
+        private IEnumerable<Symbol> FilterUniverse(IEnumerable<ETFConstituentUniverse> constituents)
         {
             _selectionDone = true;
             return constituents.Select(x => x.Symbol);

Algorithm.Framework/Selection/ETFConstituentsUniverseSelectionModel.cs

@@ -27,7 +27,7 @@ public class ETFConstituentsUniverseSelectionModel : UniverseSelectionModel
     {
         private readonly Symbol _etfSymbol;
         private readonly UniverseSettings _universeSettings;
-        private readonly Func<IEnumerable<ETFConstituentData>, IEnumerable<Symbol>> _universeFilterFunc;
+        private readonly Func<IEnumerable<ETFConstituentUniverse>, IEnumerable<Symbol>> _universeFilterFunc;
         private Universe _universe;
 
         /// <summary>
@@ -39,7 +39,7 @@ public class ETFConstituentsUniverseSelectionModel : UniverseSelectionModel
         public ETFConstituentsUniverseSelectionModel(
             Symbol etfSymbol,
             UniverseSettings universeSettings,
-            Func<IEnumerable<ETFConstituentData>, IEnumerable<Symbol>> universeFilterFunc)
+            Func<IEnumerable<ETFConstituentUniverse>, IEnumerable<Symbol>> universeFilterFunc)
         {
             _etfSymbol = etfSymbol;
             _universeSettings = universeSettings;
@@ -53,7 +53,7 @@ public ETFConstituentsUniverseSelectionModel(
         /// <param name="universeFilterFunc">Function to filter universe results</param>
         public ETFConstituentsUniverseSelectionModel(
             Symbol etfSymbol,
-            Func<IEnumerable<ETFConstituentData>, IEnumerable<Symbol>> universeFilterFunc)
+            Func<IEnumerable<ETFConstituentUniverse>, IEnumerable<Symbol>> universeFilterFunc)
             : this(etfSymbol, null, universeFilterFunc)
         { }
 
@@ -67,7 +67,7 @@ public ETFConstituentsUniverseSelectionModel(
             Symbol etfSymbol,
             UniverseSettings universeSettings = null,
             PyObject universeFilterFunc = null) :
-            this(etfSymbol, universeSettings, universeFilterFunc.ConvertPythonUniverseFilterFunction<ETFConstituentData>())
+            this(etfSymbol, universeSettings, universeFilterFunc.ConvertPythonUniverseFilterFunction<ETFConstituentUniverse>())
         { }
 
         /// <summary>
@@ -79,7 +79,7 @@ public ETFConstituentsUniverseSelectionModel(
         public ETFConstituentsUniverseSelectionModel(
             string etfTicker,
             UniverseSettings universeSettings,
-            Func<IEnumerable<ETFConstituentData>, IEnumerable<Symbol>> universeFilterFunc)
+            Func<IEnumerable<ETFConstituentUniverse>, IEnumerable<Symbol>> universeFilterFunc)
         {
             _etfSymbol = SymbolCache.TryGetSymbol(etfTicker, out var symbol)
                 && symbol.SecurityType == SecurityType.Equity
@@ -96,7 +96,7 @@ public ETFConstituentsUniverseSelectionModel(
         /// <param name="universeFilterFunc">Function to filter universe results</param>
         public ETFConstituentsUniverseSelectionModel(
             string etfTicker,
-            Func<IEnumerable<ETFConstituentData>, IEnumerable<Symbol>> universeFilterFunc)
+            Func<IEnumerable<ETFConstituentUniverse>, IEnumerable<Symbol>> universeFilterFunc)
             : this(etfTicker, null, universeFilterFunc)
         { }
 
@@ -111,7 +111,7 @@ public ETFConstituentsUniverseSelectionModel(
             string etfTicker,
             UniverseSettings universeSettings = null,
             PyObject universeFilterFunc = null) :
-            this(etfTicker, universeSettings, universeFilterFunc.ConvertPythonUniverseFilterFunction<ETFConstituentData>())
+            this(etfTicker, universeSettings, universeFilterFunc.ConvertPythonUniverseFilterFunction<ETFConstituentUniverse>())
         { }
 
         /// <summary>

Algorithm.Python/ETFConstituentUniverseFrameworkRegressionAlgorithm.py

@@ -123,7 +123,14 @@ def Initialize(self):
         spy = Symbol.Create("SPY", SecurityType.Equity, Market.USA)
 
         self.UniverseSettings.Resolution = Resolution.Hour
-        self.AddUniverse(self.Universe.ETF(spy, self.UniverseSettings, self.FilterETFConstituents))
+        universe = self.AddUniverse(self.Universe.ETF(spy, self.UniverseSettings, self.FilterETFConstituents))
+
+        historicalData = self.History(universe, 1)
+        if len(historicalData) != 1:
+            raise ValueError(f"Unexpected history count {len(historicalData)}! Expected 1");
+        for universeDataCollection in historicalData:
+            if len(universeDataCollection) < 200:
+               raise ValueError(f"Unexpected universe DataCollection count {len(universeDataCollection)}! Expected > 200");
 
     ### <summary>
     ### Filters ETF constituents
@@ -142,4 +149,4 @@ def FilterETFConstituents(self, constituents):
     ### no-op for performance
     ### </summary>
     def OnData(self, data):
-        pass
+        pass

Algorithm.Python/ETFConstituentUniverseRSIAlphaModelAlgorithm.py

@@ -63,10 +63,10 @@ def __init__(self, maxTrades=None):
     def Update(self, algorithm: QCAlgorithm, data: Slice):
         algoConstituents = []
         for barSymbol in data.Bars.Keys:
-            if not algorithm.Securities[barSymbol].Cache.HasData(ETFConstituentData):
+            if not algorithm.Securities[barSymbol].Cache.HasData(ETFConstituentUniverse):
                 continue
 
-            constituentData = algorithm.Securities[barSymbol].Cache.GetData[ETFConstituentData]()
+            constituentData = algorithm.Securities[barSymbol].Cache.GetData[ETFConstituentUniverse]()
             algoConstituents.append(constituentData)
 
         if len(algoConstituents) == 0 or len(data.Bars) == 0: