Fix warming up bug in Stochastic indicator

Algorithm.CSharp/StochasticIndicatorWarmsUpProperlyRegressionAlgorithm.cs

@@ -0,0 +1,172 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Indicators;
+using QuantConnect.Interfaces;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm that asserts Stochastic indicator, registered with a different resolution consolidator,
+    /// is warmed up properly by calling QCAlgorithm.WarmUpIndicator
+    /// </summary>
+    public class StochasticIndicatorWarmsUpProperlyRegressionAlgorithm: QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private bool _dataPointsReceived;
+        private Symbol _spy;
+        private RelativeStrengthIndex _rsi;
+        private RelativeStrengthIndex _rsiHistory;
+        private Stochastic _sto;
+        private Stochastic _stoHistory;
+
+        public override void Initialize()
+        {
+            SetStartDate(2020, 1, 1);
+            SetEndDate(2020, 2, 1);
+
+            _spy = AddEquity("SPY", Resolution.Hour).Symbol;
+
+            var dailyConsolidator = new TradeBarConsolidator(TimeSpan.FromDays(1));
+            _rsi = new RelativeStrengthIndex(14, MovingAverageType.Wilders);
+            _sto = new Stochastic("FIRST", 14, 3, 3);
+            RegisterIndicator(_spy, _rsi, dailyConsolidator);
+            RegisterIndicator(_spy, _sto, dailyConsolidator);
+
+            WarmUpIndicator(_spy, _rsi, TimeSpan.FromDays(1));
+            WarmUpIndicator(_spy, _sto, TimeSpan.FromDays(1));
+
+            _rsiHistory = new RelativeStrengthIndex(14, MovingAverageType.Wilders);
+            _stoHistory = new Stochastic("SECOND", 14, 3, 3);
+            RegisterIndicator(_spy, _rsiHistory, dailyConsolidator);
+            RegisterIndicator(_spy, _stoHistory, dailyConsolidator);
+
+            var history = History(_spy, 15, Resolution.Daily);
+            foreach (var bar in history)
+            {
+                _rsiHistory.Update(bar.EndTime, bar.Close);
+                if (_rsiHistory.Samples == 1) continue;
+
+                _stoHistory.Update(bar);
+            }
+
+            var indicators = new List<IIndicator>() { _rsi, _sto, _rsiHistory, _stoHistory };
+
+            foreach (var indicator in indicators)
+            {
+                if (!indicator.IsReady)
+                {
+                    throw new RegressionTestException($"{indicator.Name} should be ready, but it is not. Number of samples: {indicator.Samples}");
+                }
+            }
+        }
+
+        public override void OnData(Slice slice)
+        {
+            if (IsWarmingUp) return;
+
+            if (slice.ContainsKey(_spy))
+            {
+                _dataPointsReceived = true;
+
+                if (_rsi.Current.Value != _rsiHistory.Current.Value)
+                {
+                    throw new RegressionTestException($"Values of indicators differ: {_rsi.Name}: {_rsi.Current.Value} | {_rsiHistory.Name}: {_rsiHistory.Current.Value}");
+                }
+
+                if (_sto.StochK.Current.Value != _stoHistory.StochK.Current.Value)
+                {
+                    throw new RegressionTestException($"Stoch K values of indicators differ: {_sto.Name}.StochK: {_sto.StochK.Current.Value} | {_stoHistory.Name}.StochK: {_stoHistory.StochK.Current.Value}");
+                }
+
+                if (_sto.StochD.Current.Value != _stoHistory.StochD.Current.Value)
+                {
+                    throw new RegressionTestException($"Stoch D values of indicators differ: {_sto.Name}.StochD: {_sto.StochD.Current.Value} | {_stoHistory.Name}.StochD: {_stoHistory.StochD.Current.Value}");
+                }
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_dataPointsReceived)
+            {
+                throw new Exception("No data points received");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 302;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 44;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100000"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-0.016"},
+            {"Tracking Error", "0.101"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Portfolio Turnover", "0%"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.Python/StochasticIndicatorWarmsUpProperlyRegressionAlgorithm.py

@@ -0,0 +1,77 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import timedelta
+from AlgorithmImports import *
+
+### <summary>
+### Regression algorithm that asserts Stochastic indicator, registered with a different resolution consolidator,
+### is warmed up properly by calling QCAlgorithm.WarmUpIndicator
+### </summary>
+class StochasticIndicatorWarmsUpProperlyRegressionAlgorithm(QCAlgorithm):
+    def initialize(self):
+        self.set_start_date(2020, 1, 1)  # monday = holiday..
+        self.set_end_date(2020, 2, 1)
+        self.set_cash(100000)
+
+        self.data_points_received = False;
+        self.spy = self.add_equity("SPY", Resolution.HOUR).symbol
+
+        self.daily_consolidator = TradeBarConsolidator(timedelta(days=1))
+
+        self._rsi = RelativeStrengthIndex(14, MovingAverageType.WILDERS)
+        self._sto = Stochastic("FIRST", 14, 3, 3)
+        self.register_indicator(self.spy, self._rsi, self.daily_consolidator)
+        self.register_indicator(self.spy, self._sto, self.daily_consolidator)
+
+        # warm_up indicator
+        self.warm_up_indicator(self.spy, self._rsi, timedelta(days=1))
+        self.warm_up_indicator(self.spy, self._sto, timedelta(days=1))
+
+
+        self._rsi_history = RelativeStrengthIndex(14, MovingAverageType.WILDERS)
+        self._sto_history = Stochastic("SECOND", 14, 3, 3)
+        self.register_indicator(self.spy, self._rsi_history, self.daily_consolidator)
+        self.register_indicator(self.spy, self._sto_history, self.daily_consolidator)
+
+        # history warm up
+        history = self.history[TradeBar](self.spy, 15, Resolution.DAILY)
+        for bar in history:
+            self._rsi_history.update(bar.end_time, bar.close)
+            if self._rsi_history.samples == 1:
+                continue
+            self._sto_history.update(bar)
+
+        indicators = [self._rsi, self._sto, self._rsi_history, self._sto_history]
+        for indicator in indicators:
+            if not indicator.is_ready:
+                raise Exception(f"{indicator.name} should be ready, but it is not. Number of samples: {indicator.samples}")
+
+    def on_data(self, data: Slice):
+        if self.is_warming_up:
+            return
+
+        if data.contains_key(self.spy):
+            self.data_points_received = True
+            if self._rsi.current.value != self._rsi_history.current.value:
+                raise Exception(f"Values of indicators differ: {self._rsi.name}: {self._rsi.current.value} | {self._rsi_history.name}: {self._rsi_history.current.value}")
+
+            if self._sto.stoch_k.current.value != self._sto_history.stoch_k.current.value:
+                raise Exception(f"Stoch K values of indicators differ: {self._sto.name}.StochK: {self._sto.stoch_k.current.value} | {self._sto_history.name}.StochK: {self._sto_history.stoch_k.current.value}")
+
+            if self._sto.stoch_d.current.value != self._sto_history.stoch_d.current.value:
+                raise Exception(f"Stoch D values of indicators differ: {self._sto.name}.StochD: {self._sto.stoch_d.current.value} | {self._sto_history.name}.StochD: {self._sto_history.stoch_d.current.value}")
+
+    def on_end_of_algorithm(self):
+        if not self.data_points_received:
+            raise Exception("No data points received")

Algorithm/QCAlgorithm.Indicators.cs

@@ -3052,7 +3052,7 @@ public void WarmUpIndicator(IEnumerable<Symbol> symbols, IndicatorBase<Indicator
         [DocumentationAttribute(Indicators)]
         public void WarmUpIndicator(Symbol symbol, IndicatorBase<IndicatorDataPoint> indicator, TimeSpan period, Func<IBaseData, decimal> selector = null)
         {
-            var history = GetIndicatorWarmUpHistory(new[] { symbol }, indicator, period, out var identityConsolidator);
+            var history = GetIndicatorWarmUpHistory(symbol, indicator, period, out var identityConsolidator, out var historyRequest);
             if (history == Enumerable.Empty<Slice>()) return;
 
             // assign default using cast
@@ -3064,7 +3064,7 @@ public void WarmUpIndicator(Symbol symbol, IndicatorBase<IndicatorDataPoint> ind
                 indicator.Update(input);
             };
 
-            WarmUpIndicatorImpl(symbol, period, onDataConsolidated, history, identityConsolidator);
+            WarmUpIndicatorImpl(symbol, period, onDataConsolidated, history, identityConsolidator, historyRequest);
         }
 
         /// <summary>
@@ -3112,7 +3112,7 @@ public void WarmUpIndicator<T>(IEnumerable<Symbol> symbols, IndicatorBase<T> ind
         public void WarmUpIndicator<T>(Symbol symbol, IndicatorBase<T> indicator, TimeSpan period, Func<IBaseData, T> selector = null)
             where T : class, IBaseData
         {
-            var history = GetIndicatorWarmUpHistory(new[] { symbol }, indicator, period, out var identityConsolidator);
+            var history = GetIndicatorWarmUpHistory(symbol, indicator, period, out var identityConsolidator, out var historyRequest);
             if (history == Enumerable.Empty<Slice>()) return;
 
             // assign default using cast
@@ -3124,12 +3124,14 @@ public void WarmUpIndicator<T>(Symbol symbol, IndicatorBase<T> indicator, TimeSp
                 indicator.Update(selector(bar));
             };
 
-            WarmUpIndicatorImpl(symbol, period, onDataConsolidated, history, identityConsolidator);
+            WarmUpIndicatorImpl(symbol, period, onDataConsolidated, history, identityConsolidator, historyRequest);
         }
 
-        private IEnumerable<Slice> GetIndicatorWarmUpHistory(IEnumerable<Symbol> symbols, IIndicator indicator, TimeSpan timeSpan, out bool identityConsolidator)
+        private IEnumerable<Slice> GetIndicatorWarmUpHistory(Symbol symbol, IIndicator indicator, TimeSpan timeSpan, out bool identityConsolidator, out HistoryRequest historyRequest)
         {
             identityConsolidator = false;
+            historyRequest = null;
+
             if (!AssertIndicatorHasWarmupPeriod(indicator))
             {
                 return Enumerable.Empty<Slice>();
@@ -3149,7 +3151,10 @@ private IEnumerable<Slice> GetIndicatorWarmUpHistory(IEnumerable<Symbol> symbols
 
                 try
                 {
-                    return History(symbols, periods, resolution, dataNormalizationMode: GetIndicatorHistoryDataNormalizationMode(indicator));
+                    var symbols = new[] { symbol };
+                    CheckPeriodBasedHistoryRequestResolution(symbols, resolution, null);
+                    historyRequest = CreateBarCountHistoryRequests(symbols, periods, resolution, dataNormalizationMode: GetIndicatorHistoryDataNormalizationMode(indicator)).Single();
+                    return GetSlicesFromHistoryRequests(historyRequest);
                 }
                 catch (ArgumentException e)
                 {
@@ -3159,6 +3164,11 @@ private IEnumerable<Slice> GetIndicatorWarmUpHistory(IEnumerable<Symbol> symbols
             return Enumerable.Empty<Slice>();
         }
 
+        private IEnumerable<Slice> GetSlicesFromHistoryRequests(HistoryRequest historyRequest)
+        {
+            return History(historyRequest).Memoize();
+        }
+
         private bool AssertIndicatorHasWarmupPeriod(IIndicator indicator)
         {
             if (indicator is not IIndicatorWarmUpPeriodProvider)
@@ -3175,7 +3185,7 @@ private bool AssertIndicatorHasWarmupPeriod(IIndicator indicator)
             return true;
         }
 
-        private void WarmUpIndicatorImpl<T>(Symbol symbol, TimeSpan period, Action<T> handler, IEnumerable<Slice> history, bool identityConsolidator)
+        private void WarmUpIndicatorImpl<T>(Symbol symbol, TimeSpan period, Action<T> handler, IEnumerable<Slice> history, bool identityConsolidator, HistoryRequest historyRequest)
             where T : class, IBaseData
         {
             IDataConsolidator consolidator;
@@ -3227,7 +3237,7 @@ private void WarmUpIndicatorImpl<T>(Symbol symbol, TimeSpan period, Action<T> ha
             // Scan for time after we've pumped all the data through for this consolidator
             if (lastBar != null)
             {
-                consolidator.Scan(lastBar.EndTime);
+                consolidator.Scan(historyRequest.EndTimeLocal);
             }
 
             SubscriptionManager.RemoveConsolidator(symbol, consolidator);