Skip to content

Commit

Permalink
Refactor universe historical data source (QuantConnect#7837)
Browse files Browse the repository at this point in the history 
* Refactor universe historical data source

- Add new universe history API methods
- Refactor QuantBook UniverseHistory to use the universe selection
  itself instead of a given func
- Refactor and rename fundamental types
- Refactor AddUniverse API to handle universe collection data which
  holds another type internally, like fundamental

* Fix minor bug causing ApiDataProvider not to serve Bitfinex universe data

* Further improvements to add universe API

* Handle no selection function
  • Loading branch information
@Martin-Molinero
Martin-Molinero authored Mar 12, 2024
1 parent a98a502 commit 17ca8a7
Show file tree
Hide file tree
Showing 49 changed files with 955 additions and 396 deletions.
6 changes: 3 additions & 3 deletions Algorithm.CSharp/CustomDataUniverseAlgorithm.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/*
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
Expand Down Expand Up @@ -51,7 +51,7 @@ public override void Initialize()
AddUniverse<NyseTopGainers>("universe-nyse-top-gainers", Resolution.Daily, data =>
{
// define our selection criteria
return from d in data
return from NyseTopGainers d in data
// pick top 2 gainers to bet against
where d.TopGainersRank <= 2
select d.Symbol;
Expand Down Expand Up @@ -165,4 +165,4 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
}
}
}
}
}
2 changes: 1 addition & 1 deletion Algorithm.CSharp/CustomDataUniverseImmediateSelectionRegressionAlgorithm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public override void Initialize()
AddUniverse<StockDataSource>("my-stock-data-source", stockDataSource =>
{
_selected = true;
return stockDataSource.SelectMany(x => x.Symbols);
return stockDataSource.OfType<StockDataSource>().SelectMany(x => x.Symbols);
});
}

Expand Down
8 changes: 3 additions & 5 deletions Algorithm.CSharp/CustomDataUniverseRegressionAlgorithm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ public class CustomDataUniverseRegressionAlgorithm : QCAlgorithm, IRegressionAlg
new DateTime(2014, 03, 26, 0, 0, 0),
new DateTime(2014, 03, 27, 0, 0, 0),
new DateTime(2014, 03, 28, 0, 0, 0),
new DateTime(2014, 03, 29, 0, 0, 0),
new DateTime(2014, 03, 30, 0, 0, 0),
new DateTime(2014, 03, 31, 0, 0, 0)
new DateTime(2014, 03, 29, 0, 0, 0)
});

/// <summary>
Expand All @@ -56,7 +54,7 @@ public override void Initialize()
{
throw new Exception($"Unexpected selection time {Time} expected {expectedTime}");
}
return coarse.OrderByDescending(x => x.DollarVolume)
return coarse.OfType<CoarseFundamental>().OrderByDescending(x => x.DollarVolume)
.SelectMany(x => new[] {
x.Symbol,
QuantConnect.Symbol.CreateBase(typeof(CustomData), x.Symbol)})
Expand Down Expand Up @@ -107,7 +105,7 @@ public override void OnEndOfAlgorithm()
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 42611;
public long DataPoints => 42633;

/// <summary>
/// Data Points count of the algorithm history
Expand Down
59 changes: 37 additions & 22 deletions Algorithm.CSharp/DropboxBaseDataUniverseSelectionAlgorithm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,28 @@ public override void Initialize()
// Commented so regression algorithm is more sensitive
//Settings.MinimumOrderMarginPortfolioPercentage = 0.005m;

SetStartDate(2017, 07, 04);
SetStartDate(2017, 07, 06);
SetEndDate(2018, 07, 04);

AddUniverse<StockDataSource>("my-stock-data-source", stockDataSource =>
var universe = AddUniverse<StockDataSource>(stockDataSource =>
{
return stockDataSource.SelectMany(x => x.Symbols);
return stockDataSource.OfType<StockDataSource>().SelectMany(x => x.Symbols);
});

var historicalSelectionData = History(universe, 3).ToList();
if (historicalSelectionData.Count != 3)
{
throw new Exception($"Unexpected universe data count {historicalSelectionData.Count}");
}

foreach (var universeData in historicalSelectionData)
{
var stockDataSource = (StockDataSource)universeData.Single();
if (stockDataSource.Symbols.Count != 5)
{
throw new Exception($"Unexpected universe data receieved");
}
}
}

/// <summary>
Expand Down Expand Up @@ -104,7 +119,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
/// <summary>
/// Our custom data type that defines where to get and how to read our backtest and live data.
/// </summary>
class StockDataSource : BaseData
class StockDataSource : BaseDataCollection
{
private const string LiveUrl = @"https://www.dropbox.com/s/2l73mu97gcehmh7/daily-stock-picker-live.csv?dl=1";
private const string BacktestUrl = @"https://www.dropbox.com/s/ae1couew5ir3z9y/daily-stock-picker-backtest.csv?dl=1";
Expand Down Expand Up @@ -133,7 +148,7 @@ public StockDataSource()
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var url = isLiveMode ? LiveUrl : BacktestUrl;
return new SubscriptionDataSource(url, SubscriptionTransportMedium.RemoteFile);
return new SubscriptionDataSource(url, SubscriptionTransportMedium.RemoteFile, FileFormat.FoldingCollection);
}

/// <summary>
Expand Down Expand Up @@ -186,43 +201,43 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 5301;
public long DataPoints => 5282;

/// <summary>
/// Data Points count of the algorithm history
/// </summary>
public int AlgorithmHistoryDataPoints => 0;
public int AlgorithmHistoryDataPoints => 3;

/// <summary>
/// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
/// </summary>
public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
{
{"Total Orders", "6467"},
{"Total Orders", "6441"},
{"Average Win", "0.07%"},
{"Average Loss", "-0.07%"},
{"Compounding Annual Return", "14.841%"},
{"Drawdown", "10.400%"},
{"Expectancy", "0.068"},
{"Net Profit", "14.841%"},
{"Sharpe Ratio", "0.795"},
{"Sortino Ratio", "0.746"},
{"Probabilistic Sharpe Ratio", "46.586%"},
{"Compounding Annual Return", "15.505%"},
{"Drawdown", "10.500%"},
{"Expectancy", "0.070"},
{"Net Profit", "15.414%"},
{"Sharpe Ratio", "0.846"},
{"Sortino Ratio", "0.789"},
{"Probabilistic Sharpe Ratio", "48.709%"},
{"Loss Rate", "46%"},
{"Win Rate", "54%"},
{"Profit-Loss Ratio", "0.97"},
{"Profit-Loss Ratio", "0.98"},
{"Alpha", "0.008"},
{"Beta", "0.987"},
{"Annual Standard Deviation", "0.11"},
{"Annual Variance", "0.012"},
{"Information Ratio", "0.166"},
{"Information Ratio", "0.161"},
{"Tracking Error", "0.041"},
{"Treynor Ratio", "0.089"},
{"Total Fees", "$7497.26"},
{"Estimated Strategy Capacity", "$320000.00"},
{"Treynor Ratio", "0.094"},
{"Total Fees", "$7489.03"},
{"Estimated Strategy Capacity", "$220000.00"},
{"Lowest Capacity Asset", "BNO UN3IMQ2JU1YD"},
{"Portfolio Turnover", "136.11%"},
{"OrderListHash", "776d059e40150c00dd3df117239491fd"}
{"Portfolio Turnover", "136.17%"},
{"OrderListHash", "5186452592597aa6f9c94f5a237472d8"}
};
}
}
76 changes: 49 additions & 27 deletions Algorithm.CSharp/FundamentalRegressionAlgorithm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,17 @@ public class FundamentalRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmD
private const int NumberOfSymbolsFundamental = 2;

private SecurityChanges _changes = SecurityChanges.None;
private Universe _universe;

public override void Initialize()
{
UniverseSettings.Resolution = Resolution.Daily;

SetStartDate(2014, 03, 25);
SetStartDate(2014, 03, 26);
SetEndDate(2014, 04, 07);

_universe = AddUniverse(FundamentalSelectionFunction);

// before we add any symbol
AssertFundamentalUniverseData();

Expand Down Expand Up @@ -85,26 +88,45 @@ public override void Initialize()
}
}
AssertFundamentalUniverseData();

AddUniverse(FundamentalSelectionFunction);
}

private void AssertFundamentalUniverseData()
{
// Request historical fundamental data for all symbols
var history2 = History<Fundamentals>(new TimeSpan(1, 0, 0, 0)).ToList();
if (history2.Count != 1)
// we run it twice just to match the history request data point count with the python version which has 1 extra different api test/assert
for (var i = 0; i < 2; i++)
{
throw new Exception($"Unexpected {nameof(Fundamentals)} history count {history2.Count}! Expected 1");
// Request historical fundamental data for all symbols, passing the universe instance
var universeDataPerTime = History(_universe, new TimeSpan(2, 0, 0, 0)).ToList();
if (universeDataPerTime.Count != 2)
{
throw new Exception($"Unexpected {nameof(Fundamentals)} history count {universeDataPerTime.Count}! Expected 1");
}

foreach (var universeDataCollection in universeDataPerTime)
{
AssertFundamentalEnumerator(universeDataCollection, "1");
}
}
var data = history2[0].Single().Value.Data;
if (data.Count < 7000)

// Passing through the unvierse type and symbol
var enumerableOfDataDictionary = History<FundamentalUniverse>(new[] { _universe.Symbol }, 100);
foreach (var selectionCollectionForADay in enumerableOfDataDictionary)
{
AssertFundamentalEnumerator(selectionCollectionForADay[_universe.Symbol], "2");
}
}

private void AssertFundamentalEnumerator(IEnumerable<BaseData> enumerable, string caseName)
{
var dataPointCount = 0;
// note we need to cast to Fundamental type
foreach (Fundamental fundamental in enumerable)
{
throw new Exception($"Unexpected {nameof(Fundamentals)} data count {data.Count}! Expected > 7000");
dataPointCount++;
}
if (data.Any(x => x.GetType() != typeof(Fundamental)))
if (dataPointCount < 7000)
{
throw new Exception($"Unexpected {nameof(Fundamentals)} data type!");
throw new Exception($"Unexpected historical {nameof(Fundamentals)} data count {dataPointCount} case {caseName}! Expected > 7000");
}
}

Expand Down Expand Up @@ -168,12 +190,12 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 85867;
public long DataPoints => 77967;

/// <summary>
/// Data Points count of the algorithm history
/// </summary>
public virtual int AlgorithmHistoryDataPoints => 4;
public virtual int AlgorithmHistoryDataPoints => 14;

/// <summary>
/// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
Expand All @@ -183,28 +205,28 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
{"Total Orders", "2"},
{"Average Win", "0%"},
{"Average Loss", "0%"},
{"Compounding Annual Return", "-0.223%"},
{"Compounding Annual Return", "-2.572%"},
{"Drawdown", "0.100%"},
{"Expectancy", "0"},
{"Net Profit", "-0.009%"},
{"Sharpe Ratio", "-6.313"},
{"Sortino Ratio", "-8.551"},
{"Probabilistic Sharpe Ratio", "12.055%"},
{"Net Profit", "-0.093%"},
{"Sharpe Ratio", "-4.883"},
{"Sortino Ratio", "-6.653"},
{"Probabilistic Sharpe Ratio", "22.758%"},
{"Loss Rate", "0%"},
{"Win Rate", "0%"},
{"Profit-Loss Ratio", "0"},
{"Alpha", "-0.019"},
{"Beta", "0.027"},
{"Annual Standard Deviation", "0.004"},
{"Alpha", "-0.014"},
{"Beta", "0.023"},
{"Annual Standard Deviation", "0.003"},
{"Annual Variance", "0"},
{"Information Ratio", "1.749"},
{"Information Ratio", "0.597"},
{"Tracking Error", "0.095"},
{"Treynor Ratio", "-0.876"},
{"Treynor Ratio", "-0.694"},
{"Total Fees", "$2.00"},
{"Estimated Strategy Capacity", "$2200000000.00"},
{"Estimated Strategy Capacity", "$1500000000.00"},
{"Lowest Capacity Asset", "IBM R735QTJ8XC9X"},
{"Portfolio Turnover", "0.28%"},
{"OrderListHash", "490a9beb7a7b09c88db446e4fbd392cc"}
{"Portfolio Turnover", "0.30%"},
{"OrderListHash", "1e19c27ee1573978a10370b6a243edf7"}
};
}
}
2 changes: 1 addition & 1 deletion Algorithm.CSharp/FundamentalUniverseSelectionRegressionAlgorithm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public override void Initialize()
{
UniverseSettings.Resolution = Resolution.Daily;

SetStartDate(2014, 03, 25);
SetStartDate(2014, 03, 26);
SetEndDate(2014, 04, 07);

AddEquity("SPY");
Expand Down
Loading

0 comments on commit 17ca8a7

Please sign in to comment.