Skip to content

Commit

Permalink
Merge pull request QuantConnect#3748 from QuantConnect/bug-3745-fix-h…
Browse files Browse the repository at this point in the history 
…istory-request-for-cboe

Fixes failing CBOE History request by removing duplicate property
  • Loading branch information
@jaredbroad
jaredbroad authored Oct 24, 2019
2 parents 8070c81 + b97d11d commit 19fa866
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 33 deletions.
5 changes: 0 additions & 5 deletions Common/Data/Custom/CBOE/CBOE.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,6 @@ namespace QuantConnect.Data.Custom.CBOE
{
public class CBOE : TradeBar
{
/// <summary>
/// Market data type
/// </summary>
public new MarketDataType DataType;

/// <summary>
/// Creates a new instance of the object
/// </summary>
Expand Down
83 changes: 55 additions & 28 deletions Common/Python/PandasData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using QuantConnect.Util;
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
Expand All @@ -33,23 +34,22 @@ public class PandasData
private static dynamic _pandas;
private static dynamic _remapperFactory;
private readonly static HashSet<string> _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant());
private readonly static ConcurrentDictionary<Type, List<MemberInfo>> _membersByType = new ConcurrentDictionary<Type, List<MemberInfo>>();

private readonly int _levels;
private readonly bool _isCustomData;
private readonly Symbol _symbol;
private readonly Dictionary<string, Tuple<List<DateTime>, List<object>>> _series;

private readonly IEnumerable<MemberInfo> _members;
private readonly List<MemberInfo> _members;

/// <summary>
/// Gets true if this is a custom data request, false for normal QC data
/// </summary>
public bool IsCustomData => _isCustomData;
public bool IsCustomData { get; }

/// <summary>
/// Implied levels of a multi index pandas.Series (depends on the security type)
/// </summary>
public int Levels => _levels;
public int Levels { get; } = 2;

/// <summary>
/// Initializes an instance of <see cref="PandasData"/>
Expand Down Expand Up @@ -245,46 +245,60 @@ def openinterest(self):
}
}

var type = data.GetType() as Type;
_isCustomData = type.Namespace != "QuantConnect.Data.Market";
_members = Enumerable.Empty<MemberInfo>();
_symbol = (data as IBaseData)?.Symbol;
var type = data.GetType();
IsCustomData = type.Namespace != typeof(Bar).Namespace;
_members = new List<MemberInfo>();
_symbol = ((IBaseData)data).Symbol;

_levels = 2;
if (_symbol.SecurityType == SecurityType.Future) _levels = 3;
if (_symbol.SecurityType == SecurityType.Option) _levels = 5;
if (_symbol.SecurityType == SecurityType.Future) Levels = 3;
if (_symbol.SecurityType == SecurityType.Option) Levels = 5;

var columns = new List<string>
var columns = new HashSet<string>
{
"open", "high", "low", "close", "lastprice", "volume",
"askopen", "askhigh", "asklow", "askclose", "askprice", "asksize", "quantity", "suspicious",
"bidopen", "bidhigh", "bidlow", "bidclose", "bidprice", "bidsize", "exchange", "openinterest"
};

if (_isCustomData)
if (IsCustomData)
{
var keys = (data as DynamicData)?.GetStorageDictionary().Select(x => x.Key);
var keys = (data as DynamicData)?.GetStorageDictionary().ToHashSet(x => x.Key);

// C# types that are not DynamicData type
if (keys == null)
{
var members = type.GetMembers().Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property);

var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
foreach (var duplicateKey in duplicateKeys)
if (_membersByType.TryGetValue(type, out _members))
{
throw new ArgumentException($"PandasData.ctor(): More than one \'{duplicateKey}\' member was found in \'{type.FullName}\' class.");
keys = _members.ToHashSet(x => x.Name.ToLowerInvariant());
}
else
{
var members = type.GetMembers().Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property).ToList();

var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
foreach (var duplicateKey in duplicateKeys)
{
throw new ArgumentException($"PandasData.ctor(): More than one \'{duplicateKey}\' member was found in \'{type.FullName}\' class.");
}

// If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys
keys = members.ToHashSet(x => x.Name.ToLowerInvariant());
keys.ExceptWith(_baseDataProperties);
keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type));
keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type));
keys.ExceptWith(GetPropertiesNames(typeof(Tick), type));
keys.Add("value");

_members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList();
_membersByType.TryAdd(type, _members);
}

keys = members.Select(x => x.Name.ToLowerInvariant()).Except(_baseDataProperties).Concat(new[] { "value" });
_members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant()));
}

columns.Add("value");
columns.AddRange(keys);
columns.UnionWith(keys);
}

_series = columns.Distinct().ToDictionary(k => k, v => Tuple.Create(new List<DateTime>(), new List<object>()));
_series = columns.ToDictionary(k => k, v => Tuple.Create(new List<DateTime>(), new List<object>()));
}

/// <summary>
Expand All @@ -296,16 +310,16 @@ public void Add(object baseData)
foreach (var member in _members)
{
var key = member.Name.ToLowerInvariant();
var endTime = (baseData as IBaseData).EndTime;
var endTime = ((IBaseData) baseData).EndTime;
AddToSeries(key, endTime, (member as FieldInfo)?.GetValue(baseData));
AddToSeries(key, endTime, (member as PropertyInfo)?.GetValue(baseData));
}

var storage = (baseData as DynamicData)?.GetStorageDictionary();
if (storage != null)
{
var endTime = (baseData as IBaseData).EndTime;
var value = (baseData as IBaseData).Value;
var endTime = ((IBaseData) baseData).EndTime;
var value = ((IBaseData) baseData).Value;
AddToSeries("value", endTime, value);

foreach (var kvp in storage)
Expand Down Expand Up @@ -490,6 +504,19 @@ private void AddToSeries(string key, DateTime time, object input)
}
}

/// <summary>
/// Get the lower-invariant name of properties of the type that a another type is assignable from
/// </summary>
/// <param name="baseType">The type that is assignable from</param>
/// <param name="type">The type that is assignable by</param>
/// <returns>List of string. Empty list if not assignable from</returns>
private static IEnumerable<string> GetPropertiesNames(Type baseType, Type type)
{
return baseType.IsAssignableFrom(type)
? baseType.GetProperties().Select(x => x.Name.ToLowerInvariant())
: Enumerable.Empty<string>();
}

/// <summary>
/// Will wrap the provided pandas data frame using the <see cref="_remapperFactory"/>
/// </summary>
Expand Down
93 changes: 93 additions & 0 deletions Tests/Python/PandasConverterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,7 @@ public void HandlesCustomDataBars(Type type, string format)
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);

var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal));
var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions");
expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]);
Expand Down Expand Up @@ -948,6 +949,74 @@ public void HandlesCustomDataBars(Type type, string format)
}
}

[Test]
[TestCase(typeof(SubTradeBar), "SubProperty")]
[TestCase(typeof(SubSubTradeBar), "SubSubProperty")]
public void HandlesCustomDataBarsInheritsFromTradeBar(Type type, string propertyName)
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;

var config = GetSubscriptionDataConfig<Quandl>(symbol, Resolution.Daily);
dynamic custom = Activator.CreateInstance(type);

var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToStringInvariant("yyyyMMdd HH:mm")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false) as BaseData;
})
.ToArray();

// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);

using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));

var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));

var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);

for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);

var transactions = subDataFrame.loc[index][propertyName.ToLowerInvariant()].AsManagedObject(typeof(decimal));
var expected = type.GetProperty(propertyName)?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}

// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);

using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));

var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));

var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(10, count);

for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
}
}
}

private object[] SpotMarketCases => LeanDataReaderTests.SpotMarketCases;
private object[] OptionAndFuturesCases => LeanDataReaderTests.OptionAndFuturesCases;

Expand Down Expand Up @@ -1057,5 +1126,29 @@ private dynamic GetTestDataFrame(Symbol symbol)
.ToArray();
return converter.GetDataFrame(rawBars);
}

internal class SubTradeBar : TradeBar
{
public decimal SubProperty => Value;

public SubTradeBar() { }

public SubTradeBar(TradeBar tradeBar) : base(tradeBar) { }

public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) =>
new SubTradeBar((TradeBar) base.Reader(config, line, date, isLiveMode));
}

internal class SubSubTradeBar : SubTradeBar
{
public decimal SubSubProperty => Value;

public SubSubTradeBar() { }

public SubSubTradeBar(TradeBar tradeBar) : base(tradeBar) { }

public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) =>
new SubSubTradeBar((TradeBar) base.Reader(config, line, date, isLiveMode));
}
}
}

0 comments on commit 19fa866

Please sign in to comment.