Lean exchange improvements (QuantConnect#5932)

* Lean exchanges improvements

- Adding new Exchange class to avoid exchange code clash.
  Adding and updating unit test

* Add Market for MapFile API

* Self review

Algorithm.CSharp/BasicTemplateAtreyuAlgorithm.cs

@@ -124,7 +124,7 @@ public override void OnData(Slice data)
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "cb542eaaeab5eac3bcae5d915ded30da"}
+            {"OrderListHash", "867df80d1338dc526316a01e68435498"}
         };
     }
 }

Common/Data/Auxiliary/LocalDiskMapFileProvider.cs

@@ -66,7 +66,7 @@ public MapFileResolver Get(string market)
 
         private static MapFileResolver GetMapFileResolver(string market)
         {
-            var mapFileDirectory = Path.Combine(Globals.CacheDataFolder, "equity", market.ToLowerInvariant(), "map_files");
+            var mapFileDirectory = Path.Combine(Globals.CacheDataFolder, "equity", market, "map_files");
             if (!Directory.Exists(mapFileDirectory))
             {
                 // only write this message once per application instance
@@ -78,7 +78,7 @@ private static MapFileResolver GetMapFileResolver(string market)
                 }
                 return MapFileResolver.Empty;
             }
-            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory));
+            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market));
         }
     }
 }

Common/Data/Auxiliary/LocalZipMapFileProvider.cs

@@ -121,7 +121,7 @@ private MapFileResolver GetMapFileResolver(string market)
                 if (stream != null)
                 {
                     Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
-                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream));
+                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market));
                     stream.DisposeSafely();
                     return result;
                 }

Common/Data/Auxiliary/MapFile.cs

@@ -145,7 +145,7 @@ public IEnumerable<string> ToCsvLines()
         /// </summary>
         public static MapFile Read(string permtick, string market)
         {
-            return new MapFile(permtick, MapFileRow.Read(permtick, market));
+            return new MapFile(permtick, MapFileRow.Read(GetMapFilePath(permtick, market), market));
         }
 
         /// <summary>
@@ -209,16 +209,17 @@ IEnumerator IEnumerable.GetEnumerator()
         /// Reads all the map files in the specified directory
         /// </summary>
         /// <param name="mapFileDirectory">The map file directory path</param>
+        /// <param name="market">The map file market</param>
         /// <returns>An enumerable of all map files</returns>
-        public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory)
+        public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory, string market)
         {
             var mapFiles = new ConcurrentBag<MapFile>();
             Parallel.ForEach(Directory.EnumerateFiles(mapFileDirectory), file =>
             {
                 if (file.EndsWith(".csv"))
                 {
                     var permtick = Path.GetFileNameWithoutExtension(file);
-                    var fileRead = SafeMapFileRowRead(file);
+                    var fileRead = SafeMapFileRowRead(file, market);
                     mapFiles.Add(new MapFile(permtick, fileRead));
                 }
             });
@@ -228,11 +229,11 @@ public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory)
         /// <summary>
         /// Reads in the map file at the specified path, returning null if any exceptions are encountered
         /// </summary>
-        private static List<MapFileRow> SafeMapFileRowRead(string file)
+        private static List<MapFileRow> SafeMapFileRowRead(string file, string market)
         {
             try
             {
-                return MapFileRow.Read(file).ToList();
+                return MapFileRow.Read(file, market).ToList();
             }
             catch (Exception err)
             {
@@ -241,4 +242,4 @@ private static List<MapFileRow> SafeMapFileRowRead(string file)
             }
         }
     }
-}
+}

Common/Data/Auxiliary/MapFileResolver.cs

@@ -90,18 +90,9 @@ public MapFileResolver(IEnumerable<MapFile> mapFiles)
         /// <returns>The collection of map files capable of mapping equity symbols within the specified market</returns>
         public static MapFileResolver Create(string dataDirectory, string market)
         {
-            return Create(Path.Combine(dataDirectory, "equity", market.ToLowerInvariant(), "map_files"));
-        }
-
-        /// <summary>
-        /// Creates a new instance of the <see cref="MapFileResolver"/> class by reading all map files
-        /// for the specified market into memory
-        /// </summary>
-        /// <param name="mapFileDirectory">The directory containing the map files</param>
-        /// <returns>The collection of map files capable of mapping equity symbols within the specified market</returns>
-        public static MapFileResolver Create(string mapFileDirectory)
-        {
-            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory));
+            market = market.ToLowerInvariant();
+            var path = Path.Combine(dataDirectory, "equity", market, "map_files");
+            return new MapFileResolver(MapFile.GetMapFiles(path, market));
         }
 
         /// <summary>
@@ -249,4 +240,4 @@ IEnumerator IEnumerable.GetEnumerator()
 
         #endregion
     }
-}
+}

Common/Data/Auxiliary/MapFileRow.cs

@@ -15,10 +15,9 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Collections.Generic;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -45,49 +44,40 @@ public class MapFileRow : IEquatable<MapFileRow>
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFileRow"/> class.
         /// </summary>
-        public MapFileRow(DateTime date, string mappedSymbol, char primaryExchange = '\0')
-            : this(date, mappedSymbol, Exchanges.GetPrimaryExchange(primaryExchange))
+        public MapFileRow(DateTime date, string mappedSymbol, string primaryExchange, string market = QuantConnect.Market.USA)
+            : this(date, mappedSymbol, primaryExchange.GetPrimaryExchange(SecurityType.Equity, market))
         { }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFileRow"/> class.
         /// </summary>
-        public MapFileRow(DateTime date, string mappedSymbol, Exchange primaryExchange)
+        public MapFileRow(DateTime date, string mappedSymbol, Exchange primaryExchange = null)
         {
             Date = date;
             MappedSymbol = mappedSymbol.LazyToUpper();
-            PrimaryExchange = primaryExchange;
+            PrimaryExchange = primaryExchange ?? Exchange.UNKNOWN;
         }
 
         /// <summary>
         /// Reads in the map_file for the specified equity symbol
         /// </summary>
-        public static IEnumerable<MapFileRow> Read(string permtick, string market)
+        public static IEnumerable<MapFileRow> Read(string file, string market)
         {
-            var path = MapFile.GetMapFilePath(permtick, market);
-            return File.Exists(path)
-                ? Read(path)
+            return File.Exists(file)
+                ? File.ReadAllLines(file).Where(l => !string.IsNullOrWhiteSpace(l)).Select(s => Parse(s, market))
                 : Enumerable.Empty<MapFileRow>();
         }
 
-        /// <summary>
-        /// Reads in the map_file at the specified path
-        /// </summary>
-        public static IEnumerable<MapFileRow> Read(string path)
-        {
-            return File.ReadAllLines(path).Where(l => !string.IsNullOrWhiteSpace(l)).Select(Parse);
-        }
-
         /// <summary>
         /// Parses the specified line into a MapFileRow
         /// </summary>
-        public static MapFileRow Parse(string line)
+        public static MapFileRow Parse(string line, string market)
         {
             var csv = line.Split(',');
             var primaryExchange = Exchange.UNKNOWN;
             if (csv.Length == 3)
             {
-                primaryExchange = Exchanges.GetPrimaryExchange(Convert.ToChar(csv[2], CultureInfo.InvariantCulture));
+                primaryExchange = csv[2].GetPrimaryExchange(SecurityType.Equity, market);
             }
 
             return new MapFileRow(DateTime.ParseExact(csv[0], DateFormat.EightCharacter, null), csv[1], primaryExchange);
@@ -166,7 +156,7 @@ public override int GetHashCode()
         /// </summary>
         public string ToCsv()
         {
-            var encodedExchange = PrimaryExchange == Exchange.UNKNOWN? string.Empty : $",{Convert.ToChar((byte) PrimaryExchange)}";
+            var encodedExchange = PrimaryExchange == Exchange.UNKNOWN? string.Empty : $",{PrimaryExchange.Code}";
             return $"{Date.ToStringInvariant(DateFormat.EightCharacter)},{MappedSymbol.ToLowerInvariant()}{encodedExchange}";
         }
 
@@ -176,7 +166,7 @@ public string ToCsv()
         /// <returns>resulting string</returns>
         public override string ToString()
         {
-            var mainExchange = PrimaryExchange == Exchange.UNKNOWN ? string.Empty : $" - {PrimaryExchange.ToString()}";
+            var mainExchange = PrimaryExchange == Exchange.UNKNOWN ? string.Empty : $" - {PrimaryExchange}";
             return Date.ToShortDateString() + ": " + MappedSymbol + mainExchange;
         }
     }

Common/Data/Auxiliary/MapFileZipHelper.cs

@@ -36,7 +36,7 @@ public static string GetMapFileZipFileName(string market, DateTime date)
         /// <summary>
         /// Reads the zip bytes as text and parses as MapFileRows to create MapFiles
         /// </summary>
-        public static IEnumerable<MapFile> ReadMapFileZip(Stream file)
+        public static IEnumerable<MapFile> ReadMapFileZip(Stream file, string market)
         {
             if (file == null || file.Length == 0)
             {
@@ -47,20 +47,20 @@ public static IEnumerable<MapFile> ReadMapFileZip(Stream file)
                    let filename = kvp.Key
                    where filename.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase)
                    let lines = kvp.Value.Where(line => !string.IsNullOrEmpty(line))
-                   let mapFile = SafeRead(filename, lines)
+                   let mapFile = SafeRead(filename, lines, market)
                    select mapFile;
             return result;
         }
 
         /// <summary>
         /// Parses the contents as a MapFile, if error returns a new empty map file
         /// </summary>
-        private static MapFile SafeRead(string filename, IEnumerable<string> contents)
+        private static MapFile SafeRead(string filename, IEnumerable<string> contents, string market)
         {
             var permtick = Path.GetFileNameWithoutExtension(filename);
             try
             {
-                return new MapFile(permtick, contents.Select(MapFileRow.Parse));
+                return new MapFile(permtick, contents.Select(s => MapFileRow.Parse(s, market)));
             }
             catch
             {

Common/Data/Market/Tick.cs

@@ -14,13 +14,13 @@
 */
 
 using System;
-using System.Globalization;
+using ProtoBuf;
 using System.IO;
-using System.Runtime.CompilerServices;
 using Newtonsoft.Json;
-using ProtoBuf;
-using QuantConnect.Logging;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using System.Globalization;
+using System.Runtime.CompilerServices;
 
 namespace QuantConnect.Data.Market
 {
@@ -32,8 +32,8 @@ namespace QuantConnect.Data.Market
     [ProtoInclude(1000, typeof(OpenInterest))]
     public class Tick : BaseData
     {
-        private string _exchange = string.Empty;
-        private byte _exchangeCode;
+        private Exchange _exchange = QuantConnect.Exchange.UNKNOWN;
+        private string _exchangeValue;
         private uint? _parsedSaleCondition;
 
         /// <summary>
@@ -51,17 +51,21 @@ public class Tick : BaseData
         /// <summary>
         /// Exchange code this tick came from <see cref="Exchanges"/>
         /// </summary>
-        public byte ExchangeCode
+        public string ExchangeCode
         {
             get
             {
-                return _exchangeCode;
+                if (_exchange == null)
+                {
+                    _exchange = Symbol != null
+                        ? _exchangeValue.GetPrimaryExchange(Symbol.SecurityType, Symbol.ID.Market) : _exchangeValue.GetPrimaryExchange();
+                }
+                return _exchange.Code;
             }
             set
             {
-                value = Enum.IsDefined(typeof(Exchange), value) ? value : (byte)QuantConnect.Exchange.UNKNOWN;
-                _exchangeCode = value;
-                _exchange = ((Exchange) value).ToString();
+                _exchangeValue = value;
+                _exchange = null;
             }
         }
 
@@ -73,12 +77,17 @@ public string Exchange
         {
             get
             {
+                if (_exchange == null)
+                {
+                    _exchange = Symbol != null
+                        ? _exchangeValue.GetPrimaryExchange(Symbol.SecurityType, Symbol.ID.Market) : _exchangeValue.GetPrimaryExchange();
+                }
                 return _exchange;
             }
             set
             {
-                _exchange = value;
-                _exchangeCode = (byte) Exchanges.GetPrimaryExchange(_exchange);
+                _exchangeValue = value;
+                _exchange = null;
             }
         }
 
@@ -168,8 +177,8 @@ public Tick()
             Symbol = Symbol.Empty;
             TickType = TickType.Trade;
             Quantity = 0;
-            Exchange = "";
-            SaleCondition = "";
+            _exchange = QuantConnect.Exchange.UNKNOWN;
+            SaleCondition = string.Empty;
             Suspicious = false;
             BidSize = 0;
             AskSize = 0;
@@ -188,7 +197,7 @@ public Tick(Tick original)
             AskPrice = original.AskPrice;
             // directly set privates so we don't parse the exchange
             _exchange = original._exchange;
-            _exchangeCode = original._exchangeCode;
+            _exchangeValue = original._exchangeValue;
             SaleCondition = original.SaleCondition;
             Quantity = original.Quantity;
             Suspicious = original.Suspicious;