Skip to content

Commit 74d95f1

Browse files
RomazesMartin-Molinero
Romazes
and
Martin-Molinero
authored
Implement Download Project in Lean (QuantConnect#7975)
* feat: init Download proj * feat: update docker file to copy download proj * feat: parsing in cmd arguments * feat: DataDownloadConfig model feat: add config file refactor: logic of parsing config * feat: overrid ToString in DataDownloaderGetParameters * remove: not used argument command * feat: calculate downloading % (without ETA time) * fix: dockfile copy path for DownloaderDataProvider proj * feat: additional Property in .csproj * remove: DataProviderFullName prop from DataDownloadConfig * remove: extra package (ref from Configuration proj) * feat: ETA calculation of downloading data * Minor tweak * rename: namespace of downloaderDataProvider to generic style format * test: ETA from DownloaderDataProvider * fix: calculate ETA like real downloading process --------- Co-authored-by: Martin Molinero <[email protected]>
1 parent 55b0565 commit 74d95f1

12 files changed

+584
-4
lines changed

‎Common/DataDownloaderGetParameters.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,5 +63,11 @@ public DataDownloaderGetParameters(Symbol symbol, Resolution resolution, DateTim
6363
EndUtc = endUtc;
6464
TickType = tickType ?? TickType.Trade;
6565
}
66+
67+
/// <summary>
68+
/// Returns a string representation of the <see cref="DataDownloaderGetParameters"/> object.
69+
/// </summary>
70+
/// <returns>A string representing the object's properties.</returns>
71+
public override string ToString() => $"Symbol: {Symbol}, Resolution: {Resolution}, StartUtc: {StartUtc}, EndUtc: {EndUtc}, TickType: {TickType}";
6672
}
6773
}

‎Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ COPY ./Lean/Data/ /Lean/Data/
1919
COPY ./Lean/Launcher/bin/Debug/ /Lean/Launcher/bin/Debug/
2020
COPY ./Lean/Optimizer.Launcher/bin/Debug/ /Lean/Optimizer.Launcher/bin/Debug/
2121
COPY ./Lean/Report/bin/Debug/ /Lean/Report/bin/Debug/
22+
COPY ./Lean/DownloaderDataProvider/bin/Debug/ /Lean/DownloaderDataProvider/bin/Debug/
2223

2324
# Can override with '-w'
2425
WORKDIR /Lean/Launcher/bin/Debug

‎DownloaderDataProvider/DataDownloadConfig.cs

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
*/
16+
17+
using System.Globalization;
18+
using QuantConnect.Configuration;
19+
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
20+
21+
namespace QuantConnect.DownloaderDataProvider.Launcher
22+
{
23+
/// <summary>
24+
/// Represents the configuration for data download.
25+
/// </summary>
26+
public struct DataDownloadConfig
27+
{
28+
/// <summary>
29+
/// The tick type as a string.
30+
/// </summary>
31+
private string _tickType;
32+
33+
/// <summary>
34+
/// The security type as a string.
35+
/// </summary>
36+
private string _securityType;
37+
38+
/// <summary>
39+
/// The resolution as a string.
40+
/// </summary>
41+
private string _resolution;
42+
43+
/// <summary>
44+
/// The start date as a string.
45+
/// </summary>
46+
private string _startDate;
47+
48+
/// <summary>
49+
/// The end date as a string.
50+
/// </summary>
51+
private string _endDate;
52+
53+
/// <summary>
54+
/// Type of tick data to download.
55+
/// </summary>
56+
public TickType TickType { get => ParseEnum<TickType>(_tickType); }
57+
58+
/// <summary>
59+
/// Type of security for which data is to be downloaded.
60+
/// </summary>
61+
public SecurityType SecurityType { get => ParseEnum<SecurityType>(_securityType); }
62+
63+
/// <summary>
64+
/// Resolution of the downloaded data.
65+
/// </summary>
66+
public Resolution Resolution { get => ParseEnum<Resolution>(_resolution); }
67+
68+
/// <summary>
69+
/// Start date for the data download.
70+
/// </summary>
71+
public DateTime StartDate { get => DateTime.ParseExact(_startDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture); }
72+
73+
/// <summary>
74+
/// End date for the data download.
75+
/// </summary>
76+
public DateTime EndDate { get => DateTime.ParseExact(_endDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture); }
77+
78+
/// <summary>
79+
/// Market name for which the data is to be downloaded.
80+
/// </summary>
81+
public string MarketName { get; }
82+
83+
/// <summary>
84+
/// List of symbols for which data is to be downloaded.
85+
/// </summary>
86+
public List<Symbol> Symbols { get; } = new();
87+
88+
/// <summary>
89+
/// Initializes a new instance of the <see cref="DataDownloadConfig"/> struct.
90+
/// </summary>
91+
/// <param name="parameters">Dictionary containing the parameters for data download.</param>
92+
public DataDownloadConfig()
93+
{
94+
_tickType = Config.Get(DownloaderCommandArguments.CommandDataType).ToString() ?? string.Empty;
95+
_securityType = Config.Get(DownloaderCommandArguments.CommandSecurityType).ToString() ?? string.Empty;
96+
_resolution = Config.Get(DownloaderCommandArguments.CommandResolution).ToString() ?? string.Empty;
97+
98+
_startDate = Config.Get(DownloaderCommandArguments.CommandStartDate).ToString() ?? string.Empty;
99+
_endDate = Config.Get(DownloaderCommandArguments.CommandEndDate).ToString() ?? string.Empty;
100+
101+
MarketName = Config.Get(DownloaderCommandArguments.CommandMarketName).ToString()?.ToLower() ?? Market.USA;
102+
103+
if (!Market.SupportedMarkets().Contains(MarketName))
104+
{
105+
throw new ArgumentException($"The specified market '{MarketName}' is not supported. Supported markets are: {string.Join(", ", Market.SupportedMarkets())}.");
106+
}
107+
108+
foreach (var ticker in (Config.GetValue<Dictionary<string, string>>(DownloaderCommandArguments.CommandTickers))!.Keys)
109+
{
110+
Symbols.Add(Symbol.Create(ticker, SecurityType, MarketName));
111+
}
112+
}
113+
114+
/// <summary>
115+
/// Returns a string representation of the <see cref="DataDownloadConfig"/> struct.
116+
/// </summary>
117+
/// <returns>A string representation of the <see cref="DataDownloadConfig"/> struct.</returns>
118+
public override string ToString()
119+
{
120+
return $"TickType: {TickType}, " +
121+
$"SecurityType: {SecurityType}, " +
122+
$"Resolution: {Resolution}, " +
123+
$"StartDate: {StartDate:yyyyMMdd}, " +
124+
$"EndDate: {EndDate:yyyyMMdd}, " +
125+
$"MarketName: {MarketName}, " +
126+
$"Symbols: {string.Join(", ", Symbols.Select(s => s.ToString()))}";
127+
}
128+
129+
/// <summary>
130+
/// Parses a string value to an enum of type <typeparamref name="TEnum"/>.
131+
/// </summary>
132+
/// <typeparam name="TEnum">The enum type to parse to.</typeparam>
133+
/// <param name="value">The string value to parse.</param>
134+
/// <returns>The parsed enum value.</returns>
135+
private TEnum ParseEnum<TEnum>(string value) where TEnum : struct, Enum
136+
{
137+
if (!Enum.TryParse(value, true, out TEnum result) || !Enum.IsDefined(typeof(TEnum), result))
138+
{
139+
throw new ArgumentException($"Invalid {typeof(TEnum).Name} specified. Please provide a valid {typeof(TEnum).Name}.");
140+
}
141+
142+
return result;
143+
}
144+
}
145+
}

‎DownloaderDataProvider/DownloaderDataProviderArgumentParser.cs

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
*
8+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using QuantConnect.Configuration;
18+
using McMaster.Extensions.CommandLineUtils;
19+
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
20+
21+
namespace QuantConnect.DownloaderDataProvider.Launcher;
22+
23+
public static class DownloaderDataProviderArgumentParser
24+
{
25+
private const string ApplicationName = "QuantConnect.DownloaderDataProvider.exe";
26+
private const string ApplicationDescription = "Welcome to Lean Downloader Data Provider! 🚀 Easily download historical data from various sources with our user-friendly application. Start exploring financial data effortlessly!";
27+
private const string ApplicationHelpText = "Hm...";
28+
29+
private static readonly List<CommandLineOption> Options = new List<CommandLineOption>
30+
{
31+
new CommandLineOption(DownloaderCommandArguments.CommandDownloaderDataDownloader, CommandOptionType.SingleValue),
32+
new CommandLineOption(DownloaderCommandArguments.CommandDataType, CommandOptionType.SingleValue),
33+
new CommandLineOption(DownloaderCommandArguments.CommandTickers, CommandOptionType.MultipleValue),
34+
new CommandLineOption(DownloaderCommandArguments.CommandSecurityType, CommandOptionType.SingleValue),
35+
new CommandLineOption(DownloaderCommandArguments.CommandMarketName, CommandOptionType.SingleValue),
36+
new CommandLineOption(DownloaderCommandArguments.CommandResolution, CommandOptionType.SingleValue),
37+
new CommandLineOption(DownloaderCommandArguments.CommandStartDate, CommandOptionType.SingleValue),
38+
new CommandLineOption(DownloaderCommandArguments.CommandEndDate, CommandOptionType.SingleValue)
39+
};
40+
41+
/// <summary>
42+
/// Parses the command-line arguments and returns a dictionary containing parsed values.
43+
/// </summary>
44+
/// <param name="args">An array of command-line arguments.</param>
45+
/// <returns>A dictionary containing parsed values from the command-line arguments.</returns>
46+
/// <remarks>
47+
/// The <paramref name="args"/> parameter should contain the command-line arguments to be parsed.
48+
/// The method uses the ApplicationParser class to parse the arguments based on the ApplicationName,
49+
/// ApplicationDescription, ApplicationHelpText, and Options properties.
50+
/// </remarks>
51+
public static Dictionary<string, object> ParseArguments(string[] args)
52+
{
53+
return ApplicationParser.Parse(ApplicationName, ApplicationDescription, ApplicationHelpText, args, Options);
54+
}
55+
}

‎DownloaderDataProvider/Models/Constants/DownloaderCommandArguments.cs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
*
8+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
namespace QuantConnect.DownloaderDataProvider.Launcher.Models.Constants
18+
{
19+
public sealed class DownloaderCommandArguments
20+
{
21+
public const string CommandDownloaderDataDownloader = "data-downloader";
22+
23+
public const string CommandDataType = "data-type";
24+
25+
public const string CommandTickers = "tickers";
26+
27+
public const string CommandSecurityType = "security-type";
28+
29+
public const string CommandMarketName = "market";
30+
31+
public const string CommandResolution = "resolution";
32+
33+
public const string CommandStartDate = "start-date";
34+
35+
public const string CommandEndDate = "end-date";
36+
}
37+
}

0 commit comments

Comments
 (0)