Skip to content

Commit 87a6545

Browse files
RomazesRomazes
authored
Feature: Universe Data Downloader (#8612)
* feat: parsing of universe file path for different security types * fix: market position in parsing of universe file refactor: add example of universes file path in xml description method * test:feat: parsing of universe path * rename: return variable in ParseUniversePath * feat: extension to create history request internally * feat: entity DataUniverseDownloaderGetParameters * config: downloaderDataProvider project * feat: create DataUniverseDownloadConfig in download project * feat: run universe downloader in DownloaderProvider * test:fix: missing using test:remove: parameter ctor of DataDownloadConfig * refactor: run Universe downloader in Main * feat: GetUniverseFileName by processingDate * refactor: CreateDataUniverseDownloaderGetParameters with using Start and EndDate refactor: exception message in DataUniverseDownloadConfig * feat: new out argument in TryParsePath's LeanData * feat: RunUniverseDownloader in DownloadDataProvider * feat: DerivativeUniverseData to write csv file feat: extension to download universe data with different parameters * remove: log warn which path when get universe file path * refactor: DataUniverseDownloaderGetParameters feat: run Download Universes file in DownloaderDataProvider internally * refactor: overload TryParsePath only with 4 parameters * remove: editconfig file in DownloaderDataProvider * remove: validation on null parameters * rename: typo DateType to DataType * refactor: use endDate in ctor DataUniverseDownloaderGetParameters refactor: use UnderlyingSymbol from base class Symbol property remove: CreateDataUniverseDownloaderGetParameters() refactor: put LoadSymbol in base class clean/remove: DataUniverseDownloadConfig * refactor: use Time.EachTradeableDay in CreateDataDownloaderGetParameters remove: CheckMarketOpenStatus * fix: RunUniverseDownloader with EndUtc in DownloaderDataProvider * refactor: GetUniverseFullFilePath * remove: validate historyData like Any * fix: use date from parsing path in DownloaderDataProvider * feat: download QuoteBar for Universe file feat: handle QuoteBar in DerivativeUniverseData refactor: filter option in final dictionary * revert: CreateHistoryRequest() in DownloaderExtensions * feat: use optional parameter SecurityExchangeHours in DataUniverseDownloaderGetParameters refactor: DataUniverseDownloaderGetParameters refactor: use Log.Debug instead Log.Trace to prevent spamming * fix: ordering of download universe data revert: parameter ctor in DataDownloadConfig revert:test: use parameter ctor instead of Config.Set * clean: extra `usings` * fix: Get DataType in DataDownloadConfig feat: new ctor in BaseDataDownloadConfig
1 parent d00ef1c commit 87a6545

File tree

14 files changed

+745
-195
lines changed

14 files changed

+745
-195
lines changed

‎Common/Data/UniverseSelection/DerivativeUniverseData.cs

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
*/
16+
17+
using System;
18+
using QuantConnect.Data.Market;
19+
20+
namespace QuantConnect.Data.UniverseSelection;
21+
22+
/// <summary>
23+
/// Represents derivative market data including trade and open interest information.
24+
/// </summary>
25+
public class DerivativeUniverseData
26+
{
27+
private readonly Symbol _symbol;
28+
private decimal _open;
29+
private decimal _high;
30+
private decimal _low;
31+
private decimal _close;
32+
private decimal _volume;
33+
private decimal? _openInterest;
34+
35+
/// <summary>
36+
/// Initializes a new instance of <see cref="DerivativeUniverseData"/> using open interest data.
37+
/// </summary>
38+
/// <param name="openInterest">The open interest data.</param>
39+
public DerivativeUniverseData(OpenInterest openInterest)
40+
{
41+
_symbol = openInterest.Symbol;
42+
_openInterest = openInterest.Value;
43+
}
44+
45+
/// <summary>
46+
/// Initializes a new instance of <see cref="DerivativeUniverseData"/> using trade bar data.
47+
/// </summary>
48+
/// <param name="tradeBar">The trade bar data.</param>
49+
public DerivativeUniverseData(TradeBar tradeBar)
50+
{
51+
_symbol = tradeBar.Symbol;
52+
_open = tradeBar.Open;
53+
_high = tradeBar.High;
54+
_low = tradeBar.Low;
55+
_close = tradeBar.Close;
56+
_volume = tradeBar.Volume;
57+
}
58+
59+
/// <summary>
60+
/// Initializes a new instance of <see cref="DerivativeUniverseData"/> using quote bar data.
61+
/// </summary>
62+
/// <param name="quoteBar">The quote bar data.</param>
63+
public DerivativeUniverseData(QuoteBar quoteBar)
64+
{
65+
_symbol = quoteBar.Symbol;
66+
_open = quoteBar.Open;
67+
_high = quoteBar.High;
68+
_low = quoteBar.Low;
69+
_close = quoteBar.Close;
70+
}
71+
72+
/// <summary>
73+
/// Updates the instance with new trade bar data.
74+
/// </summary>
75+
/// <param name="tradeBar">The new trade bar data.</param>
76+
/// <exception cref="ArgumentNullException">Thrown when tradeBar is null.</exception>
77+
public void UpdateByTradeBar(TradeBar tradeBar)
78+
{
79+
// If price data has already been initialized (likely from a QuoteBar)
80+
if (_open != 0 || _high != 0 || _low != 0 || _close != 0)
81+
{
82+
_volume = tradeBar.Volume;
83+
return;
84+
}
85+
86+
_open = tradeBar.Open;
87+
_high = tradeBar.High;
88+
_low = tradeBar.Low;
89+
_close = tradeBar.Close;
90+
}
91+
92+
/// <summary>
93+
/// Updates the instance with new quote bar data.
94+
/// </summary>
95+
/// <param name="quoteBar">The new quote bar data.</param>
96+
public void UpdateByQuoteBar(QuoteBar quoteBar)
97+
{
98+
_open = quoteBar.Open;
99+
_high = quoteBar.High;
100+
_low = quoteBar.Low;
101+
_close = quoteBar.Close;
102+
}
103+
104+
/// <summary>
105+
/// Updates the instance with new open interest data.
106+
/// </summary>
107+
/// <param name="openInterest">The new open interest data.</param>
108+
/// <exception cref="ArgumentNullException">Thrown when openInterest is null.</exception>
109+
public void UpdateByOpenInterest(OpenInterest openInterest)
110+
{
111+
_openInterest = openInterest.Value;
112+
}
113+
114+
/// <summary>
115+
/// Converts the current data to a CSV format string.
116+
/// </summary>
117+
/// <returns>A CSV formatted string representing the data.</returns>
118+
public string ToCsv()
119+
{
120+
return OptionUniverse.ToCsv(_symbol, _open, _high, _low, _close, _volume, _openInterest, null, NullGreeks.Instance);
121+
}
122+
}

‎Common/Data/UniverseSelection/OptionUniverse.cs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -180,12 +180,22 @@ public OptionUniverse(OptionUniverse other)
180180
/// <returns>String URL of source file.</returns>
181181
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
182182
{
183-
var path = LeanData.GenerateUniversesDirectory(Globals.DataFolder, config.Symbol);
184-
path = Path.Combine(path, $"{date:yyyyMMdd}.csv");
185-
183+
var path = GetUniverseFullFilePath(config.Symbol, date);
186184
return new SubscriptionDataSource(path, SubscriptionTransportMedium.LocalFile, FileFormat.FoldingCollection);
187185
}
188186

187+
/// <summary>
188+
/// Generates the file path for a universe data file based on the given symbol and date.
189+
/// Optionally, creates the directory if it does not exist.
190+
/// </summary>
191+
/// <param name="symbol">The financial symbol for which the universe file is generated.</param>
192+
/// <param name="date">The date associated with the universe file.</param>
193+
/// <returns>The full file path to the universe data file.</returns>
194+
public static string GetUniverseFullFilePath(Symbol symbol, DateTime date)
195+
{
196+
return Path.Combine(LeanData.GenerateUniversesDirectory(Globals.DataFolder, symbol), $"{date:yyyyMMdd}.csv");
197+
}
198+
189199
/// <summary>
190200
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
191201
/// each time it is called.

‎Common/Data/UniverseSelection/UniverseExtensions.cs

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,11 @@
1414
*/
1515

1616
using System;
17+
using System.IO;
1718
using System.Linq;
19+
using QuantConnect.Logging;
20+
using QuantConnect.Data.Market;
21+
using System.Collections.Generic;
1822

1923
namespace QuantConnect.Data.UniverseSelection
2024
{
@@ -142,5 +146,81 @@ public static Symbol CreateSymbol(SecurityType securityType, string market, stri
142146

143147
return new Symbol(sid, ticker);
144148
}
149+
150+
/// <summary>
151+
/// Processes the universe download based on parameters.
152+
/// </summary>
153+
/// <param name="dataDownloader">The data downloader instance.</param>
154+
/// <param name="universeDownloadParameters">The parameters for universe downloading.</param>
155+
public static void RunUniverseDownloader(IDataDownloader dataDownloader, DataUniverseDownloaderGetParameters universeDownloadParameters)
156+
{
157+
var universeDataBySymbol = new Dictionary<Symbol, DerivativeUniverseData>();
158+
foreach (var (processingDate, universeDownloaderParameters) in universeDownloadParameters.CreateDataDownloaderGetParameters())
159+
{
160+
universeDataBySymbol.Clear();
161+
162+
foreach (var downloaderParameters in universeDownloaderParameters)
163+
{
164+
Log.Debug($"{nameof(UniverseExtensions)}.{nameof(RunUniverseDownloader)}:Generating universe for {downloaderParameters.Symbol} on {processingDate:yyyy/MM/dd}");
165+
166+
var historyData = dataDownloader.Get(downloaderParameters);
167+
168+
if (historyData == null)
169+
{
170+
Log.Debug($"{nameof(UniverseExtensions)}.{nameof(RunUniverseDownloader)}: No data available for the following parameters: {universeDownloadParameters}");
171+
continue;
172+
}
173+
174+
foreach (var baseData in historyData)
175+
{
176+
switch (baseData)
177+
{
178+
case TradeBar tradeBar:
179+
if (!universeDataBySymbol.TryAdd(tradeBar.Symbol, new(tradeBar)))
180+
{
181+
universeDataBySymbol[tradeBar.Symbol].UpdateByTradeBar(tradeBar);
182+
}
183+
break;
184+
case OpenInterest openInterest:
185+
if (!universeDataBySymbol.TryAdd(openInterest.Symbol, new(openInterest)))
186+
{
187+
universeDataBySymbol[openInterest.Symbol].UpdateByOpenInterest(openInterest);
188+
}
189+
break;
190+
case QuoteBar quoteBar:
191+
if (!universeDataBySymbol.TryAdd(quoteBar.Symbol, new(quoteBar)))
192+
{
193+
universeDataBySymbol[quoteBar.Symbol].UpdateByQuoteBar(quoteBar);
194+
}
195+
break;
196+
default:
197+
throw new InvalidOperationException($"{nameof(UniverseExtensions)}.{nameof(RunUniverseDownloader)}: Unexpected data type encountered.");
198+
}
199+
}
200+
}
201+
202+
if (universeDataBySymbol.Count == 0)
203+
{
204+
continue;
205+
}
206+
207+
using var writer = new StreamWriter(universeDownloadParameters.GetUniverseFileName(processingDate));
208+
209+
writer.WriteLine($"#{OptionUniverse.CsvHeader}");
210+
211+
// Write option data, sorted by contract type (Call/Put), strike price, expiration date, and then by full ID
212+
foreach (var universeData in universeDataBySymbol
213+
.OrderBy(x => x.Key.Underlying != null)
214+
.ThenBy(d => d.Key.SecurityType.IsOption() ? d.Key.ID.OptionRight : 0)
215+
.ThenBy(d => d.Key.SecurityType.IsOption() ? d.Key.ID.StrikePrice : 0)
216+
.ThenBy(d => d.Key.ID.Date)
217+
.ThenBy(d => d.Key.ID))
218+
{
219+
writer.WriteLine(universeData.Value.ToCsv());
220+
}
221+
222+
Log.Trace($"{nameof(UniverseExtensions)}.{nameof(RunUniverseDownloader)}:Generated for {universeDownloadParameters.Symbol} on {processingDate:yyyy/MM/dd} with {universeDataBySymbol.Count} entries");
223+
}
224+
}
145225
}
146226
}

‎Common/DataUniverseDownloaderGetParameters.cs

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using System;
17+
using System.Linq;
18+
using QuantConnect.Util;
19+
using QuantConnect.Securities;
20+
using System.Collections.Generic;
21+
using QuantConnect.Data.UniverseSelection;
22+
23+
namespace QuantConnect;
24+
25+
/// <summary>
26+
/// Represents the parameters required for downloading universe data.
27+
/// </summary>
28+
public sealed class DataUniverseDownloaderGetParameters : DataDownloaderGetParameters
29+
{
30+
/// <summary>
31+
/// The initialized instance of the security exchange hours.
32+
/// </summary>
33+
private readonly SecurityExchangeHours _securityExchangeHours;
34+
35+
/// <summary>
36+
/// The tick types supported for universe data.
37+
/// </summary>
38+
private readonly TickType[] UniverseTickTypes = { TickType.Quote, TickType.Trade, TickType.OpenInterest };
39+
40+
/// <summary>
41+
/// Gets the underlying symbol associated with the universe.
42+
/// </summary>
43+
public Symbol UnderlyingSymbol { get => Symbol.HasUnderlying ? Symbol.Underlying : Symbol.Empty; }
44+
45+
/// <summary>
46+
/// Initializes a new instance of the <see cref="DataUniverseDownloaderGetParameters"/> class.
47+
/// </summary>
48+
/// <param name="canonicalSymbol">The canonical symbol for the data request.</param>
49+
/// <param name="startDate">The start date for the data request.</param>
50+
/// <param name="endDate">The end date for the data request.</param>
51+
/// <param name="securityExchangeHours">The security exchange hours for this symbol</param>
52+
/// <exception cref="ArgumentException">Thrown when the provided symbol is not canonical.</exception>
53+
public DataUniverseDownloaderGetParameters(Symbol canonicalSymbol, DateTime startDate, DateTime endDate, SecurityExchangeHours securityExchangeHours = default)
54+
: base(
55+
canonicalSymbol.IsCanonical() ? canonicalSymbol : throw new ArgumentException("DataUniverseDownloaderGetParameters: Symbol must be canonical.", nameof(canonicalSymbol)),
56+
Resolution.Daily,
57+
startDate,
58+
endDate)
59+
{
60+
_securityExchangeHours = securityExchangeHours ?? MarketHoursDatabase.FromDataFolder().GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType);
61+
62+
EndUtc = EndUtc.ConvertToUtc(_securityExchangeHours.TimeZone);
63+
StartUtc = StartUtc.ConvertToUtc(_securityExchangeHours.TimeZone);
64+
}
65+
66+
/// <summary>
67+
/// Gets the file name where the universe data will be saved.
68+
/// </summary>
69+
/// <param name="processingDate">The date for which the file name is generated.</param>
70+
/// <returns>The universe file name.</returns>
71+
public string GetUniverseFileName(DateTime processingDate)
72+
{
73+
return OptionUniverse.GetUniverseFullFilePath(Symbol, processingDate);
74+
}
75+
76+
/// <summary>
77+
/// Creates data download parameters for each day in the range.
78+
/// </summary>
79+
public IEnumerable<(DateTime, IEnumerable<DataDownloaderGetParameters>)> CreateDataDownloaderGetParameters()
80+
{
81+
foreach (var processingDate in Time.EachTradeableDay(_securityExchangeHours, StartUtc, EndUtc))
82+
{
83+
var processingDateUtc = processingDate.ConvertToUtc(_securityExchangeHours.TimeZone);
84+
85+
var requests = new List<DataDownloaderGetParameters>(3);
86+
87+
if (UnderlyingSymbol != Symbol.Empty)
88+
{
89+
requests.Add(new(UnderlyingSymbol, Resolution, processingDateUtc, processingDateUtc.AddDays(1), TickType.Trade));
90+
}
91+
92+
requests.AddRange(UniverseTickTypes.Select(tickType => new DataDownloaderGetParameters(Symbol, Resolution, processingDateUtc, processingDateUtc.AddDays(1), tickType)));
93+
94+
yield return (processingDate, requests);
95+
}
96+
}
97+
}

0 commit comments

Comments
 (0)