Adjust lower-resolution fill-forwarded strict end-timed daily bars

Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketDailyAlgorithm.cs

@@ -36,40 +36,40 @@ public class BasicTemplateFuturesWithExtendedMarketDailyAlgorithm : BasicTemplat
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public override long DataPoints => 14896;
+        public override long DataPoints => 14884;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
         /// </summary>
         public override Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Orders", "36"},
+            {"Total Orders", "32"},
             {"Average Win", "0.33%"},
-            {"Average Loss", "-0.03%"},
-            {"Compounding Annual Return", "0.102%"},
+            {"Average Loss", "-0.04%"},
+            {"Compounding Annual Return", "0.110%"},
             {"Drawdown", "0.300%"},
-            {"Expectancy", "0.171"},
+            {"Expectancy", "0.184"},
             {"Start Equity", "1000000"},
-            {"End Equity", "1001024.4"},
-            {"Net Profit", "0.102%"},
-            {"Sharpe Ratio", "-1.702"},
-            {"Sortino Ratio", "-0.836"},
-            {"Probabilistic Sharpe Ratio", "14.653%"},
-            {"Loss Rate", "89%"},
-            {"Win Rate", "11%"},
-            {"Profit-Loss Ratio", "9.54"},
+            {"End Equity", "1001108"},
+            {"Net Profit", "0.111%"},
+            {"Sharpe Ratio", "-1.688"},
+            {"Sortino Ratio", "-0.772"},
+            {"Probabilistic Sharpe Ratio", "14.944%"},
+            {"Loss Rate", "88%"},
+            {"Win Rate", "12%"},
+            {"Profit-Loss Ratio", "8.47"},
             {"Alpha", "-0.007"},
             {"Beta", "0.002"},
             {"Annual Standard Deviation", "0.004"},
             {"Annual Variance", "0"},
             {"Information Ratio", "-1.353"},
             {"Tracking Error", "0.089"},
-            {"Treynor Ratio", "-4.126"},
-            {"Total Fees", "$80.60"},
+            {"Treynor Ratio", "-4.099"},
+            {"Total Fees", "$72.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "ES VRJST036ZY0X"},
-            {"Portfolio Turnover", "0.97%"},
-            {"OrderListHash", "52c852d720692fab1e12212b2aba03d4"}
+            {"Portfolio Turnover", "0.87%"},
+            {"OrderListHash", "ef59fd5e4a7ae483a60d25736cf5d2d8"}
         };
     }
 }

Algorithm.CSharp/StrictEndTimeLowerResolutionFillForwardRegressionAlgorithm.cs

@@ -0,0 +1,253 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting fill forwarded data behavior for consolidators and indicators.
+    /// 1. Test that the on-consolidated event is not called for fill forwarded data in identity and higher period consolidators
+    /// 2. Test that the intra-day fill-forwarded data is not fed to indicators
+    /// </summary>
+    public class StrictEndTimeLowerResolutionFillForwardRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Security _aapl;
+
+        private BaseData _lastNonFilledForwardedData;
+        private int _dataCount;
+        private int _indicatorUpdateCount;
+
+        protected virtual bool ExtendedMarketHours => false;
+
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 30);
+
+            Settings.DailyPreciseEndTime = true;
+
+            // Fill forward resolution will be minute
+            AddEquity("SPY", Resolution.Minute);
+            _aapl = AddEquity("AAPL", Resolution.Daily, extendedMarketHours: ExtendedMarketHours);
+
+            var tradableDates = QuantConnect.Time.EachTradeableDayInTimeZone(_aapl.Exchange.Hours, StartDate, EndDate,
+                _aapl.Exchange.TimeZone, _aapl.IsExtendedMarketHours).ToList();
+
+            TestIdentityConsolidator(tradableDates);
+            TestHigherPeriodConsolidator(tradableDates);
+            TestIndicator(tradableDates);
+        }
+
+        private void TestIdentityConsolidator(List<DateTime> tradableDates)
+        {
+            var i = 0;
+            var consolidator = Consolidate<TradeBar>(_aapl.Symbol, TimeSpan.FromDays(1), (bar) =>
+            {
+                var expectedDate = tradableDates[i++];
+                var schedule = LeanData.GetDailyCalendar(expectedDate.AddDays(1), _aapl.Exchange, false);
+
+                if (bar.Time != schedule.Start || bar.EndTime != schedule.End)
+                {
+                    throw new RegressionTestException($"Unexpected consolidated bar time. " +
+                        $"Expected: [{schedule.Start} - {schedule.End}], Actual: [{bar.Time} - {bar.EndTime}]");
+                }
+
+                Debug($"Consolidated (identity) :: {bar}");
+            });
+
+            if (consolidator is not IdentityDataConsolidator<TradeBar>)
+            {
+                throw new RegressionTestException($"Unexpected consolidator type. " +
+                    $"Expected {typeof(IdentityDataConsolidator<TradeBar>)} but was {consolidator.GetType()}");
+            }
+        }
+
+        private void TestHigherPeriodConsolidator(List<DateTime> tradableDates)
+        {
+            var i = 0;
+            // Add a consolidator to assert that fill forward data is not used
+            Consolidate<TradeBar>(_aapl.Symbol, TimeSpan.FromDays(2), (bar) =>
+            {
+                var expectedStartDate = tradableDates[i++];
+                var startDateSchedule = LeanData.GetDailyCalendar(expectedStartDate.AddDays(1), _aapl.Exchange, false);
+
+                var expectedStartTime = startDateSchedule.Start;
+                var expectedEndTime = expectedStartTime.AddDays(2);
+
+                if (bar.Time != expectedStartTime || bar.EndTime != expectedEndTime)
+                {
+                    throw new RegressionTestException($"Unexpected consolidated bar time. " +
+                        $"Expected: [{expectedStartTime} - {expectedEndTime}], Actual: [{bar.Time} - {bar.EndTime}]");
+                }
+
+                if (tradableDates[i] == expectedStartDate.AddDays(1))
+                {
+                    i++;
+                }
+
+                Debug($"Consolidated (2 days) :: {bar}");
+            });
+        }
+
+        private void TestIndicator(List<DateTime> tradableDates)
+        {
+            var i = 0;
+            EMA(_aapl.Symbol, 3, Resolution.Daily).Updated += (sender, data) =>
+            {
+                _indicatorUpdateCount++;
+
+                var expectedEndTime = _aapl.Exchange.Hours.GetNextMarketClose(tradableDates[i++], false);
+                if (data.EndTime != expectedEndTime)
+                {
+                    throw new RegressionTestException($"Unexpected EMA time. Expected: {expectedEndTime}, Actual: {data.EndTime}");
+                }
+
+                Debug($"EMA :: [{data.EndTime}] {data}");
+            };
+        }
+
+        public override void OnData(Slice slice)
+        {
+            if (slice.TryGetValue(_aapl.Symbol, out var data))
+            {
+                var baseData = data as BaseData;
+                if (!baseData.IsFillForward)
+                {
+                    _lastNonFilledForwardedData = baseData;
+                }
+
+                var timeInExchangeTz = UtcTime.ConvertFromUtc(_aapl.Exchange.TimeZone);
+                var daySchedule = LeanData.GetDailyCalendar(timeInExchangeTz, _aapl.Exchange, false);
+
+                if (timeInExchangeTz == daySchedule.End)
+                {
+                    if (baseData.IsFillForward)
+                    {
+                        throw new RegressionTestException("End of day data should not be fill forward for daily subscription when data is available");
+                    }
+                }
+                else
+                {
+                    if (!baseData.IsFillForward
+                        || _lastNonFilledForwardedData == null
+                        || _lastNonFilledForwardedData.Time.Date != baseData.Time.Date
+                        || _lastNonFilledForwardedData.EndTime.Date != baseData.EndTime.Date)
+                    {
+                        throw new RegressionTestException("Data should be fill forward to minute resolution during the day");
+                    }
+                }
+
+                _dataCount++;
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            var tradableDates = QuantConnect.Time.EachTradeableDay(_aapl, StartDate.AddDays(1), EndDate, ExtendedMarketHours);
+            var tradableDatesCount = 1;
+            var expectedDataCount = 1; // One for the first day
+            foreach (var date in tradableDates)
+            {
+                tradableDatesCount++;
+                var hours = _aapl.Exchange.Hours.GetMarketHours(date);
+                foreach (var segment in hours.Segments)
+                {
+                    if (ExtendedMarketHours || segment.State == MarketHoursState.Market)
+                    {
+                        expectedDataCount += (int)(segment.End - segment.Start).TotalMinutes;
+                    }
+                }
+            }
+
+            if (_dataCount != expectedDataCount)
+            {
+                throw new RegressionTestException($"Unexpected data count. Expected: {expectedDataCount}, Actual: {_dataCount}");
+            }
+
+            if (_indicatorUpdateCount != tradableDatesCount)
+            {
+                throw new RegressionTestException($"Unexpected indicator update count. Expected: {tradableDatesCount}, Actual: {_indicatorUpdateCount}");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public virtual long DataPoints => 20805;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100000"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-7.12"},
+            {"Tracking Error", "0.109"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Portfolio Turnover", "0%"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.CSharp/StrictEndTimeLowerResolutionFillForwardWithExtendedMarketHoursRegressionAlgorithm.cs

@@ -0,0 +1,29 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting fill forwarded data behavior for consolidators and indicators.
+    /// 1. Test that the on-consolidated event is not called for fill forwarded data in identity and higher period consolidators
+    /// 2. Test that the intra-day fill-forwarded data is not fed to indicators
+    /// </summary>
+    public class StrictEndTimeLowerResolutionFillForwardWithExtendedMarketHoursRegressionAlgorithm : StrictEndTimeLowerResolutionFillForwardRegressionAlgorithm
+    {
+        protected override bool ExtendedMarketHours => true;
+
+        public override long DataPoints => 30495;
+    }
+}

Common/Util/LeanData.cs

@@ -1469,7 +1469,15 @@ public static bool UseDailyStrictEndTimes(IAlgorithmSettings settings, BaseDataR
         /// </summary>
         public static bool UseDailyStrictEndTimes(IAlgorithmSettings settings, Type dataType, Symbol symbol, TimeSpan increment, SecurityExchangeHours exchangeHours)
         {
-            return UseDailyStrictEndTimes(dataType) && UseStrictEndTime(settings.DailyPreciseEndTime, symbol, increment, exchangeHours);
+            return UseDailyStrictEndTimes(settings.DailyPreciseEndTime, dataType, symbol, increment, exchangeHours);
+        }
+
+        /// <summary>
+        /// Helper method to determine if we should use strict end time
+        /// </summary>
+        public static bool UseDailyStrictEndTimes(bool dailyStrictEndTimeEnabled, Type dataType, Symbol symbol, TimeSpan increment, SecurityExchangeHours exchangeHours)
+        {
+            return UseDailyStrictEndTimes(dataType) && UseStrictEndTime(dailyStrictEndTimeEnabled, symbol, increment, exchangeHours);
         }
 
         /// <summary>

Engine/DataFeeds/Enumerators/FillForwardEnumerator.cs

@@ -357,7 +357,6 @@ protected virtual bool RequiresFillForwardData(TimeSpan fillForwardResolution, B
                 var startTime = (_useStrictEndTime && item.Period > Time.OneHour) ? item.Start : RoundDown(item.Start, item.Period);
                 var potentialBarEndTime = startTime.ConvertToUtc(Exchange.TimeZone) + item.Period;
 
-
                 // to avoid duality it's necessary to compare potentialBarEndTime with
                 // next.EndTime calculated as Time + resolution,
                 // and both should be based on the same TZ (for example UTC)
@@ -381,7 +380,22 @@ protected virtual bool RequiresFillForwardData(TimeSpan fillForwardResolution, B
                         if (_useStrictEndTime)
                         {
                             // TODO: what about extended market hours
-                            expectedPeriod = Exchange.Hours.RegularMarketDuration;
+                            // NOTE: Not using Exchange.Hours.RegularMarketDuration so we can handle things like early closes.
+
+                            // The earliest start time would be endTime - regularMarketDuration,
+                            // we use that as the potential time to get the exchange hours.
+                            // We don't use directly nextFillForwardBarStartTime because there might be cases where there are
+                            // adjacent extended and regular market hours segments that might cause the calendar start to be
+                            // in the previous date, and if it's an extended hours-only date like a Sunday for futures,
+                            // the market duration would be zero.
+                            var marketHoursDateTime = potentialBarEndTimeInExchangeTZ - Exchange.Hours.RegularMarketDuration;
+                            // That potential start is even before the calendar start, so we use the calendar start
+                            if (marketHoursDateTime < item.Start)
+                            {
+                                marketHoursDateTime = item.Start;
+                            }
+                            var marketHours = Exchange.Hours.GetMarketHours(marketHoursDateTime);
+                            expectedPeriod = marketHours.MarketDuration;
                         }
                         fillForward.Time = (potentialBarEndTime - expectedPeriod).ConvertFromUtc(Exchange.TimeZone);
                         fillForward.EndTime = potentialBarEndTimeInExchangeTZ;
@@ -435,7 +449,7 @@ private IEnumerable<CalendarInfo> GetReferenceDateIntervals(DateTime previousEnd
             }
             else
             {
-                yield return new (marketOpen, resolution);
+                yield return new(marketOpen, resolution);
             }
         }